2024-11-24 20:46:16,332 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-24 20:46:16,345 main DEBUG Took 0.010645 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 20:46:16,345 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 20:46:16,345 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 20:46:16,346 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 20:46:16,347 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:46:16,356 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 20:46:16,371 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:46:16,372 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:46:16,373 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:46:16,373 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:46:16,374 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:46:16,375 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:46:16,376 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:46:16,376 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:46:16,377 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:46:16,377 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:46:16,378 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:46:16,379 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:46:16,379 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:46:16,380 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:46:16,380 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:46:16,381 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:46:16,381 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:46:16,382 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:46:16,382 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:46:16,383 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:46:16,383 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:46:16,383 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:46:16,384 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:46:16,385 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 20:46:16,385 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:46:16,386 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 20:46:16,387 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 20:46:16,389 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 20:46:16,392 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 20:46:16,392 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 20:46:16,394 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 20:46:16,395 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 20:46:16,404 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 20:46:16,406 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 20:46:16,407 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 20:46:16,408 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 20:46:16,408 main DEBUG createAppenders(={Console}) 2024-11-24 20:46:16,409 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-24 20:46:16,409 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-24 20:46:16,410 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-24 20:46:16,411 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 20:46:16,411 main DEBUG OutputStream closed 2024-11-24 20:46:16,411 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 20:46:16,412 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 20:46:16,412 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-24 20:46:16,491 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 20:46:16,493 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 20:46:16,494 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 20:46:16,495 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 20:46:16,495 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 20:46:16,495 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 20:46:16,496 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 20:46:16,496 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 20:46:16,496 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 20:46:16,496 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 20:46:16,497 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 20:46:16,497 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 20:46:16,497 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 20:46:16,498 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 20:46:16,498 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 20:46:16,498 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 20:46:16,498 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 20:46:16,499 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 20:46:16,501 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 20:46:16,501 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-24 20:46:16,502 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 20:46:16,502 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-24T20:46:16,767 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c 2024-11-24 20:46:16,770 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 20:46:16,770 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T20:46:16,779 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-24T20:46:16,801 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-24T20:46:16,816 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/cluster_32fb3c48-f3a2-7f88-64a5-636927e4ea42, deleteOnExit=true 2024-11-24T20:46:16,816 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-24T20:46:16,817 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/test.cache.data in system properties and HBase conf 2024-11-24T20:46:16,818 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/hadoop.tmp.dir in system properties and HBase conf 2024-11-24T20:46:16,818 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/hadoop.log.dir in system properties and HBase conf 2024-11-24T20:46:16,819 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-24T20:46:16,820 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-24T20:46:16,821 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-24T20:46:16,911 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T20:46:16,996 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T20:46:17,000 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-24T20:46:17,001 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-24T20:46:17,001 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-24T20:46:17,002 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T20:46:17,002 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-24T20:46:17,003 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-24T20:46:17,003 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T20:46:17,003 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T20:46:17,004 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-24T20:46:17,004 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/nfs.dump.dir in system properties and HBase conf 2024-11-24T20:46:17,005 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/java.io.tmpdir in system properties and HBase conf 2024-11-24T20:46:17,005 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T20:46:17,005 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-24T20:46:17,006 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-24T20:46:18,222 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-24T20:46:18,294 INFO [Time-limited test {}] log.Log(170): Logging initialized @2751ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-24T20:46:18,358 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T20:46:18,411 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T20:46:18,430 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T20:46:18,431 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T20:46:18,432 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T20:46:18,443 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T20:46:18,446 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/hadoop.log.dir/,AVAILABLE} 2024-11-24T20:46:18,447 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T20:46:18,618 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bbf242f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/java.io.tmpdir/jetty-localhost-39965-hadoop-hdfs-3_4_1-tests_jar-_-any-11578793166357425759/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T20:46:18,625 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:39965} 2024-11-24T20:46:18,625 INFO [Time-limited test {}] server.Server(415): Started @3082ms 2024-11-24T20:46:19,190 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T20:46:19,199 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T20:46:19,200 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T20:46:19,201 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T20:46:19,201 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T20:46:19,202 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b8e8cdd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/hadoop.log.dir/,AVAILABLE} 2024-11-24T20:46:19,202 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ef5f873{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T20:46:19,317 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3dfb1097{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/java.io.tmpdir/jetty-localhost-39251-hadoop-hdfs-3_4_1-tests_jar-_-any-8040797134681812327/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T20:46:19,318 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@16142ff{HTTP/1.1, (http/1.1)}{localhost:39251} 2024-11-24T20:46:19,318 INFO [Time-limited test {}] server.Server(415): Started @3775ms 2024-11-24T20:46:19,367 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T20:46:20,328 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/cluster_32fb3c48-f3a2-7f88-64a5-636927e4ea42/data/data2/current/BP-171709779-172.17.0.2-1732481177570/current, will proceed with Du for space computation calculation, 2024-11-24T20:46:20,328 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/cluster_32fb3c48-f3a2-7f88-64a5-636927e4ea42/data/data1/current/BP-171709779-172.17.0.2-1732481177570/current, will proceed with Du for space computation calculation, 2024-11-24T20:46:20,354 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T20:46:20,397 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3638a12b289ce6b3 with lease ID 0xdbf8a867e00ad2ef: Processing first storage report for DS-6a47c115-3646-4421-9ff7-0ca9cf144cbe from datanode DatanodeRegistration(127.0.0.1:40053, datanodeUuid=fd6f659b-0aaf-4b1f-84d2-c3f2d7a13354, infoPort=39053, infoSecurePort=0, ipcPort=43075, storageInfo=lv=-57;cid=testClusterID;nsid=1281814308;c=1732481177570) 2024-11-24T20:46:20,398 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3638a12b289ce6b3 with lease ID 0xdbf8a867e00ad2ef: from storage DS-6a47c115-3646-4421-9ff7-0ca9cf144cbe node DatanodeRegistration(127.0.0.1:40053, datanodeUuid=fd6f659b-0aaf-4b1f-84d2-c3f2d7a13354, infoPort=39053, infoSecurePort=0, ipcPort=43075, storageInfo=lv=-57;cid=testClusterID;nsid=1281814308;c=1732481177570), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-24T20:46:20,399 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3638a12b289ce6b3 with lease ID 0xdbf8a867e00ad2ef: Processing first storage report for DS-18d056e9-be3e-40b3-b040-e17122c33be0 from datanode DatanodeRegistration(127.0.0.1:40053, datanodeUuid=fd6f659b-0aaf-4b1f-84d2-c3f2d7a13354, infoPort=39053, infoSecurePort=0, ipcPort=43075, storageInfo=lv=-57;cid=testClusterID;nsid=1281814308;c=1732481177570) 2024-11-24T20:46:20,399 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3638a12b289ce6b3 with lease ID 0xdbf8a867e00ad2ef: from storage DS-18d056e9-be3e-40b3-b040-e17122c33be0 node DatanodeRegistration(127.0.0.1:40053, datanodeUuid=fd6f659b-0aaf-4b1f-84d2-c3f2d7a13354, infoPort=39053, infoSecurePort=0, ipcPort=43075, storageInfo=lv=-57;cid=testClusterID;nsid=1281814308;c=1732481177570), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T20:46:20,446 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c 2024-11-24T20:46:20,537 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/cluster_32fb3c48-f3a2-7f88-64a5-636927e4ea42/zookeeper_0, clientPort=63551, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/cluster_32fb3c48-f3a2-7f88-64a5-636927e4ea42/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/cluster_32fb3c48-f3a2-7f88-64a5-636927e4ea42/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T20:46:20,549 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63551 2024-11-24T20:46:20,559 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T20:46:20,563 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T20:46:20,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741825_1001 (size=7) 2024-11-24T20:46:21,201 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a with version=8 2024-11-24T20:46:21,201 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/hbase-staging 2024-11-24T20:46:21,301 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-24T20:46:21,536 INFO [Time-limited test {}] client.ConnectionUtils(128): master/20b981ef5c27:0 server-side Connection retries=45 2024-11-24T20:46:21,545 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T20:46:21,546 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T20:46:21,552 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T20:46:21,552 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T20:46:21,553 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T20:46:21,742 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T20:46:21,806 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T20:46:21,816 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T20:46:21,820 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T20:46:21,850 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 19827 (auto-detected) 2024-11-24T20:46:21,851 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-24T20:46:21,878 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38339 2024-11-24T20:46:21,909 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38339 connecting to ZooKeeper ensemble=127.0.0.1:63551 2024-11-24T20:46:22,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:383390x0, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T20:46:22,046 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38339-0x1016e98df0e0000 connected 2024-11-24T20:46:22,155 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T20:46:22,157 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T20:46:22,166 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38339-0x1016e98df0e0000, quorum=127.0.0.1:63551, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T20:46:22,171 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a, hbase.cluster.distributed=false 2024-11-24T20:46:22,200 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38339-0x1016e98df0e0000, quorum=127.0.0.1:63551, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T20:46:22,206 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38339 2024-11-24T20:46:22,207 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38339 2024-11-24T20:46:22,211 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38339 2024-11-24T20:46:22,213 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38339 2024-11-24T20:46:22,216 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38339 2024-11-24T20:46:22,328 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/20b981ef5c27:0 server-side Connection retries=45 2024-11-24T20:46:22,330 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T20:46:22,330 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T20:46:22,331 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T20:46:22,332 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T20:46:22,332 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T20:46:22,335 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T20:46:22,339 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T20:46:22,344 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34625 2024-11-24T20:46:22,347 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34625 connecting to ZooKeeper ensemble=127.0.0.1:63551 2024-11-24T20:46:22,349 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T20:46:22,356 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T20:46:22,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:346250x0, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T20:46:22,375 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:346250x0, quorum=127.0.0.1:63551, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T20:46:22,376 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34625-0x1016e98df0e0001 connected 2024-11-24T20:46:22,381 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T20:46:22,392 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T20:46:22,396 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34625-0x1016e98df0e0001, quorum=127.0.0.1:63551, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T20:46:22,404 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34625-0x1016e98df0e0001, quorum=127.0.0.1:63551, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T20:46:22,408 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34625 2024-11-24T20:46:22,409 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34625 2024-11-24T20:46:22,412 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34625 2024-11-24T20:46:22,415 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34625 2024-11-24T20:46:22,416 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34625 2024-11-24T20:46:22,438 DEBUG [M:0;20b981ef5c27:38339 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;20b981ef5c27:38339 2024-11-24T20:46:22,440 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/20b981ef5c27,38339,1732481181363 2024-11-24T20:46:22,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38339-0x1016e98df0e0000, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T20:46:22,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34625-0x1016e98df0e0001, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T20:46:22,462 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38339-0x1016e98df0e0000, quorum=127.0.0.1:63551, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/20b981ef5c27,38339,1732481181363 2024-11-24T20:46:22,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34625-0x1016e98df0e0001, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T20:46:22,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38339-0x1016e98df0e0000, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:46:22,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34625-0x1016e98df0e0001, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:46:22,502 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38339-0x1016e98df0e0000, quorum=127.0.0.1:63551, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T20:46:22,505 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/20b981ef5c27,38339,1732481181363 from backup master directory 2024-11-24T20:46:22,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34625-0x1016e98df0e0001, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T20:46:22,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38339-0x1016e98df0e0000, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/20b981ef5c27,38339,1732481181363 2024-11-24T20:46:22,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38339-0x1016e98df0e0000, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T20:46:22,519 WARN [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T20:46:22,519 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=20b981ef5c27,38339,1732481181363 2024-11-24T20:46:22,521 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-24T20:46:22,523 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-24T20:46:22,595 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/hbase.id] with ID: 450c21d0-94cf-4eb1-984f-e674d08bba31 2024-11-24T20:46:22,595 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/.tmp/hbase.id 2024-11-24T20:46:22,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741826_1002 (size=42) 2024-11-24T20:46:22,619 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/.tmp/hbase.id]:[hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/hbase.id] 2024-11-24T20:46:22,688 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T20:46:22,695 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T20:46:22,717 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-11-24T20:46:22,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34625-0x1016e98df0e0001, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:46:22,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38339-0x1016e98df0e0000, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:46:22,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741827_1003 (size=196) 2024-11-24T20:46:22,976 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T20:46:22,979 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T20:46:22,996 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:23,002 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T20:46:23,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741828_1004 (size=1189) 2024-11-24T20:46:23,061 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/MasterData/data/master/store 2024-11-24T20:46:23,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741829_1005 (size=34) 2024-11-24T20:46:23,488 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T20:46:23,490 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:46:23,492 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T20:46:23,492 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T20:46:23,492 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T20:46:23,493 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T20:46:23,494 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T20:46:23,494 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T20:46:23,495 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732481183491Disabling compacts and flushes for region at 1732481183491Disabling writes for close at 1732481183494 (+3 ms)Writing region close event to WAL at 1732481183494Closed at 1732481183494 2024-11-24T20:46:23,497 WARN [master/20b981ef5c27:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/MasterData/data/master/store/.initializing 2024-11-24T20:46:23,498 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/MasterData/WALs/20b981ef5c27,38339,1732481181363 2024-11-24T20:46:23,506 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T20:46:23,522 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=20b981ef5c27%2C38339%2C1732481181363, suffix=, logDir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/MasterData/WALs/20b981ef5c27,38339,1732481181363, archiveDir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/MasterData/oldWALs, maxLogs=10 2024-11-24T20:46:23,545 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/MasterData/WALs/20b981ef5c27,38339,1732481181363/20b981ef5c27%2C38339%2C1732481181363.1732481183526, exclude list is [], retry=0 2024-11-24T20:46:23,561 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40053,DS-6a47c115-3646-4421-9ff7-0ca9cf144cbe,DISK] 2024-11-24T20:46:23,563 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-24T20:46:23,598 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/MasterData/WALs/20b981ef5c27,38339,1732481181363/20b981ef5c27%2C38339%2C1732481181363.1732481183526 2024-11-24T20:46:23,599 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39053:39053)] 2024-11-24T20:46:23,600 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T20:46:23,600 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:46:23,603 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:46:23,604 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:46:23,635 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:46:23,656 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T20:46:23,658 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:46:23,661 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T20:46:23,661 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:46:23,664 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T20:46:23,665 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:46:23,666 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:46:23,666 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:46:23,669 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T20:46:23,669 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:46:23,670 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:46:23,670 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:46:23,673 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T20:46:23,673 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:46:23,674 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:46:23,674 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:46:23,677 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:46:23,679 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:46:23,684 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:46:23,685 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:46:23,689 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T20:46:23,694 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T20:46:23,699 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T20:46:23,700 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73603925, jitterRate=0.09678395092487335}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T20:46:23,708 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732481183615Initializing all the Stores at 1732481183617 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732481183617Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481183618 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481183618Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481183618Cleaning up temporary data from old regions at 1732481183685 (+67 ms)Region opened successfully at 1732481183707 (+22 ms) 2024-11-24T20:46:23,709 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T20:46:23,740 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@491b3393, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=20b981ef5c27/172.17.0.2:0 2024-11-24T20:46:23,779 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T20:46:23,791 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T20:46:23,791 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T20:46:23,794 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T20:46:23,795 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-24T20:46:23,800 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-24T20:46:23,801 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T20:46:23,824 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T20:46:23,832 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38339-0x1016e98df0e0000, quorum=127.0.0.1:63551, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T20:46:23,879 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T20:46:23,882 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T20:46:23,885 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38339-0x1016e98df0e0000, quorum=127.0.0.1:63551, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T20:46:23,921 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T20:46:23,925 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T20:46:23,931 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38339-0x1016e98df0e0000, quorum=127.0.0.1:63551, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T20:46:23,953 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T20:46:23,954 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38339-0x1016e98df0e0000, quorum=127.0.0.1:63551, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T20:46:23,963 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T20:46:23,979 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38339-0x1016e98df0e0000, quorum=127.0.0.1:63551, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T20:46:23,990 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T20:46:24,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34625-0x1016e98df0e0001, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T20:46:24,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38339-0x1016e98df0e0000, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T20:46:24,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38339-0x1016e98df0e0000, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:46:24,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34625-0x1016e98df0e0001, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:46:24,008 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=20b981ef5c27,38339,1732481181363, sessionid=0x1016e98df0e0000, setting cluster-up flag (Was=false) 2024-11-24T20:46:24,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34625-0x1016e98df0e0001, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:46:24,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38339-0x1016e98df0e0000, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:46:24,069 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T20:46:24,071 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=20b981ef5c27,38339,1732481181363 2024-11-24T20:46:24,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34625-0x1016e98df0e0001, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:46:24,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38339-0x1016e98df0e0000, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:46:24,128 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T20:46:24,130 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=20b981ef5c27,38339,1732481181363 2024-11-24T20:46:24,137 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T20:46:24,223 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T20:46:24,227 INFO [RS:0;20b981ef5c27:34625 {}] regionserver.HRegionServer(746): ClusterId : 450c21d0-94cf-4eb1-984f-e674d08bba31 2024-11-24T20:46:24,230 DEBUG [RS:0;20b981ef5c27:34625 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T20:46:24,234 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T20:46:24,240 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T20:46:24,246 DEBUG [RS:0;20b981ef5c27:34625 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T20:46:24,246 DEBUG [RS:0;20b981ef5c27:34625 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T20:46:24,246 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 20b981ef5c27,38339,1732481181363 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T20:46:24,259 DEBUG [RS:0;20b981ef5c27:34625 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T20:46:24,260 DEBUG [RS:0;20b981ef5c27:34625 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e1f3549, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=20b981ef5c27/172.17.0.2:0 2024-11-24T20:46:24,261 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/20b981ef5c27:0, corePoolSize=5, maxPoolSize=5 2024-11-24T20:46:24,262 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/20b981ef5c27:0, corePoolSize=5, maxPoolSize=5 2024-11-24T20:46:24,262 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/20b981ef5c27:0, corePoolSize=5, maxPoolSize=5 2024-11-24T20:46:24,262 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/20b981ef5c27:0, corePoolSize=5, maxPoolSize=5 2024-11-24T20:46:24,262 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/20b981ef5c27:0, corePoolSize=10, maxPoolSize=10 2024-11-24T20:46:24,262 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:46:24,263 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/20b981ef5c27:0, corePoolSize=2, maxPoolSize=2 2024-11-24T20:46:24,263 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:46:24,272 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732481214272 2024-11-24T20:46:24,272 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T20:46:24,273 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T20:46:24,274 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T20:46:24,276 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T20:46:24,279 DEBUG [RS:0;20b981ef5c27:34625 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;20b981ef5c27:34625 2024-11-24T20:46:24,280 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T20:46:24,280 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:46:24,280 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T20:46:24,281 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T20:46:24,281 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T20:46:24,280 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T20:46:24,283 INFO [RS:0;20b981ef5c27:34625 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T20:46:24,283 INFO [RS:0;20b981ef5c27:34625 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T20:46:24,284 DEBUG [RS:0;20b981ef5c27:34625 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T20:46:24,285 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T20:46:24,287 INFO [RS:0;20b981ef5c27:34625 {}] regionserver.HRegionServer(2659): reportForDuty to master=20b981ef5c27,38339,1732481181363 with port=34625, startcode=1732481182291 2024-11-24T20:46:24,300 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T20:46:24,301 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T20:46:24,302 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T20:46:24,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741831_1007 (size=1321) 2024-11-24T20:46:24,307 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T20:46:24,307 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T20:46:24,309 DEBUG [RS:0;20b981ef5c27:34625 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T20:46:24,310 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/20b981ef5c27:0:becomeActiveMaster-HFileCleaner.large.0-1732481184309,5,FailOnTimeoutGroup] 2024-11-24T20:46:24,311 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/20b981ef5c27:0:becomeActiveMaster-HFileCleaner.small.0-1732481184310,5,FailOnTimeoutGroup] 2024-11-24T20:46:24,311 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T20:46:24,311 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T20:46:24,311 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T20:46:24,312 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a 2024-11-24T20:46:24,313 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T20:46:24,313 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T20:46:24,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741832_1008 (size=32) 2024-11-24T20:46:24,356 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43851, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T20:46:24,363 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38339 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 20b981ef5c27,34625,1732481182291 2024-11-24T20:46:24,366 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38339 {}] master.ServerManager(517): Registering regionserver=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:24,387 DEBUG [RS:0;20b981ef5c27:34625 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a 2024-11-24T20:46:24,387 DEBUG [RS:0;20b981ef5c27:34625 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45873 2024-11-24T20:46:24,388 DEBUG [RS:0;20b981ef5c27:34625 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T20:46:24,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38339-0x1016e98df0e0000, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T20:46:24,402 DEBUG [RS:0;20b981ef5c27:34625 {}] zookeeper.ZKUtil(111): regionserver:34625-0x1016e98df0e0001, quorum=127.0.0.1:63551, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/20b981ef5c27,34625,1732481182291 2024-11-24T20:46:24,402 WARN [RS:0;20b981ef5c27:34625 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T20:46:24,402 INFO [RS:0;20b981ef5c27:34625 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T20:46:24,403 DEBUG [RS:0;20b981ef5c27:34625 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/WALs/20b981ef5c27,34625,1732481182291 2024-11-24T20:46:24,405 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [20b981ef5c27,34625,1732481182291] 2024-11-24T20:46:24,431 INFO [RS:0;20b981ef5c27:34625 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T20:46:24,450 INFO [RS:0;20b981ef5c27:34625 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T20:46:24,456 INFO [RS:0;20b981ef5c27:34625 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T20:46:24,457 INFO [RS:0;20b981ef5c27:34625 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T20:46:24,458 INFO [RS:0;20b981ef5c27:34625 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T20:46:24,464 INFO [RS:0;20b981ef5c27:34625 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T20:46:24,466 INFO [RS:0;20b981ef5c27:34625 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T20:46:24,467 DEBUG [RS:0;20b981ef5c27:34625 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:46:24,467 DEBUG [RS:0;20b981ef5c27:34625 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:46:24,467 DEBUG [RS:0;20b981ef5c27:34625 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:46:24,467 DEBUG [RS:0;20b981ef5c27:34625 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:46:24,468 DEBUG [RS:0;20b981ef5c27:34625 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:46:24,468 DEBUG [RS:0;20b981ef5c27:34625 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/20b981ef5c27:0, corePoolSize=2, maxPoolSize=2 2024-11-24T20:46:24,468 DEBUG [RS:0;20b981ef5c27:34625 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:46:24,468 DEBUG [RS:0;20b981ef5c27:34625 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:46:24,468 DEBUG [RS:0;20b981ef5c27:34625 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:46:24,469 DEBUG [RS:0;20b981ef5c27:34625 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:46:24,469 DEBUG [RS:0;20b981ef5c27:34625 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:46:24,469 DEBUG [RS:0;20b981ef5c27:34625 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/20b981ef5c27:0, corePoolSize=1, maxPoolSize=1 2024-11-24T20:46:24,469 DEBUG [RS:0;20b981ef5c27:34625 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/20b981ef5c27:0, corePoolSize=3, maxPoolSize=3 2024-11-24T20:46:24,469 DEBUG [RS:0;20b981ef5c27:34625 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0, corePoolSize=3, maxPoolSize=3 2024-11-24T20:46:24,475 INFO [RS:0;20b981ef5c27:34625 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T20:46:24,475 INFO [RS:0;20b981ef5c27:34625 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T20:46:24,475 INFO [RS:0;20b981ef5c27:34625 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T20:46:24,476 INFO [RS:0;20b981ef5c27:34625 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T20:46:24,476 INFO [RS:0;20b981ef5c27:34625 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T20:46:24,476 INFO [RS:0;20b981ef5c27:34625 {}] hbase.ChoreService(168): Chore ScheduledChore name=20b981ef5c27,34625,1732481182291-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T20:46:24,496 INFO [RS:0;20b981ef5c27:34625 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T20:46:24,498 INFO [RS:0;20b981ef5c27:34625 {}] hbase.ChoreService(168): Chore ScheduledChore name=20b981ef5c27,34625,1732481182291-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T20:46:24,498 INFO [RS:0;20b981ef5c27:34625 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T20:46:24,499 INFO [RS:0;20b981ef5c27:34625 {}] regionserver.Replication(171): 20b981ef5c27,34625,1732481182291 started 2024-11-24T20:46:24,524 INFO [RS:0;20b981ef5c27:34625 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T20:46:24,524 INFO [RS:0;20b981ef5c27:34625 {}] regionserver.HRegionServer(1482): Serving as 20b981ef5c27,34625,1732481182291, RpcServer on 20b981ef5c27/172.17.0.2:34625, sessionid=0x1016e98df0e0001 2024-11-24T20:46:24,526 DEBUG [RS:0;20b981ef5c27:34625 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T20:46:24,526 DEBUG [RS:0;20b981ef5c27:34625 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 20b981ef5c27,34625,1732481182291 2024-11-24T20:46:24,526 DEBUG [RS:0;20b981ef5c27:34625 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '20b981ef5c27,34625,1732481182291' 2024-11-24T20:46:24,527 DEBUG [RS:0;20b981ef5c27:34625 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T20:46:24,528 DEBUG [RS:0;20b981ef5c27:34625 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T20:46:24,530 DEBUG [RS:0;20b981ef5c27:34625 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T20:46:24,530 DEBUG [RS:0;20b981ef5c27:34625 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T20:46:24,530 DEBUG [RS:0;20b981ef5c27:34625 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 20b981ef5c27,34625,1732481182291 2024-11-24T20:46:24,530 DEBUG [RS:0;20b981ef5c27:34625 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '20b981ef5c27,34625,1732481182291' 2024-11-24T20:46:24,530 DEBUG [RS:0;20b981ef5c27:34625 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T20:46:24,534 DEBUG [RS:0;20b981ef5c27:34625 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T20:46:24,535 DEBUG [RS:0;20b981ef5c27:34625 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T20:46:24,536 INFO [RS:0;20b981ef5c27:34625 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T20:46:24,536 INFO [RS:0;20b981ef5c27:34625 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T20:46:24,641 INFO [RS:0;20b981ef5c27:34625 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T20:46:24,644 INFO [RS:0;20b981ef5c27:34625 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=20b981ef5c27%2C34625%2C1732481182291, suffix=, logDir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/WALs/20b981ef5c27,34625,1732481182291, archiveDir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/oldWALs, maxLogs=32 2024-11-24T20:46:24,660 DEBUG [RS:0;20b981ef5c27:34625 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/WALs/20b981ef5c27,34625,1732481182291/20b981ef5c27%2C34625%2C1732481182291.1732481184647, exclude list is [], retry=0 2024-11-24T20:46:24,665 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40053,DS-6a47c115-3646-4421-9ff7-0ca9cf144cbe,DISK] 2024-11-24T20:46:24,670 INFO [RS:0;20b981ef5c27:34625 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/WALs/20b981ef5c27,34625,1732481182291/20b981ef5c27%2C34625%2C1732481182291.1732481184647 2024-11-24T20:46:24,671 DEBUG [RS:0;20b981ef5c27:34625 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39053:39053)] 2024-11-24T20:46:24,742 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:46:24,747 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T20:46:24,751 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T20:46:24,751 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:46:24,753 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T20:46:24,753 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T20:46:24,756 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T20:46:24,757 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:46:24,758 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T20:46:24,758 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T20:46:24,761 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T20:46:24,761 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:46:24,762 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T20:46:24,763 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T20:46:24,765 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T20:46:24,765 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:46:24,766 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T20:46:24,767 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T20:46:24,768 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/hbase/meta/1588230740 2024-11-24T20:46:24,769 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/hbase/meta/1588230740 2024-11-24T20:46:24,773 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T20:46:24,774 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T20:46:24,775 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T20:46:24,778 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T20:46:24,782 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T20:46:24,783 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68540449, jitterRate=0.021332278847694397}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T20:46:24,786 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732481184743Initializing all the Stores at 1732481184746 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732481184746Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732481184746Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481184746Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732481184747 (+1 ms)Cleaning up temporary data from old regions at 1732481184774 (+27 ms)Region opened successfully at 1732481184786 (+12 ms) 2024-11-24T20:46:24,786 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T20:46:24,787 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T20:46:24,787 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T20:46:24,787 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T20:46:24,787 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T20:46:24,789 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T20:46:24,789 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732481184786Disabling compacts and flushes for region at 1732481184786Disabling writes for close at 1732481184787 (+1 ms)Writing region close event to WAL at 1732481184788 (+1 ms)Closed at 1732481184789 (+1 ms) 2024-11-24T20:46:24,793 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T20:46:24,793 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T20:46:24,799 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T20:46:24,806 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T20:46:24,809 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T20:46:24,963 DEBUG [20b981ef5c27:38339 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-24T20:46:24,975 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:24,981 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 20b981ef5c27,34625,1732481182291, state=OPENING 2024-11-24T20:46:25,005 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T20:46:25,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38339-0x1016e98df0e0000, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:46:25,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34625-0x1016e98df0e0001, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:46:25,017 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T20:46:25,017 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T20:46:25,018 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T20:46:25,020 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=20b981ef5c27,34625,1732481182291}] 2024-11-24T20:46:25,195 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T20:46:25,198 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47959, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T20:46:25,211 INFO [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T20:46:25,211 INFO [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T20:46:25,212 INFO [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T20:46:25,215 INFO [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=20b981ef5c27%2C34625%2C1732481182291.meta, suffix=.meta, logDir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/WALs/20b981ef5c27,34625,1732481182291, archiveDir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/oldWALs, maxLogs=32 2024-11-24T20:46:25,236 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/WALs/20b981ef5c27,34625,1732481182291/20b981ef5c27%2C34625%2C1732481182291.meta.1732481185218.meta, exclude list is [], retry=0 2024-11-24T20:46:25,241 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40053,DS-6a47c115-3646-4421-9ff7-0ca9cf144cbe,DISK] 2024-11-24T20:46:25,244 INFO [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/WALs/20b981ef5c27,34625,1732481182291/20b981ef5c27%2C34625%2C1732481182291.meta.1732481185218.meta 2024-11-24T20:46:25,244 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39053:39053)] 2024-11-24T20:46:25,245 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T20:46:25,247 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T20:46:25,249 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T20:46:25,254 INFO [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T20:46:25,258 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T20:46:25,259 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:46:25,259 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T20:46:25,259 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T20:46:25,262 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T20:46:25,264 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T20:46:25,264 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:46:25,265 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T20:46:25,265 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T20:46:25,267 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T20:46:25,267 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:46:25,268 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T20:46:25,268 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T20:46:25,270 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T20:46:25,270 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:46:25,271 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T20:46:25,272 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T20:46:25,273 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T20:46:25,274 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:46:25,274 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T20:46:25,275 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T20:46:25,276 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/hbase/meta/1588230740 2024-11-24T20:46:25,279 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/hbase/meta/1588230740 2024-11-24T20:46:25,282 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T20:46:25,282 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T20:46:25,283 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T20:46:25,286 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T20:46:25,287 INFO [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61123604, jitterRate=-0.08918732404708862}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T20:46:25,288 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T20:46:25,289 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732481185260Writing region info on filesystem at 1732481185260Initializing all the Stores at 1732481185262 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732481185262Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732481185262Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481185262Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732481185262Cleaning up temporary data from old regions at 1732481185282 (+20 ms)Running coprocessor post-open hooks at 1732481185288 (+6 ms)Region opened successfully at 1732481185289 (+1 ms) 2024-11-24T20:46:25,296 INFO [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732481185187 2024-11-24T20:46:25,308 DEBUG [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T20:46:25,308 INFO [RS_OPEN_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T20:46:25,310 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:25,313 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 20b981ef5c27,34625,1732481182291, state=OPEN 2024-11-24T20:46:25,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38339-0x1016e98df0e0000, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T20:46:25,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34625-0x1016e98df0e0001, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T20:46:25,440 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T20:46:25,440 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T20:46:25,440 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:25,451 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T20:46:25,451 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=20b981ef5c27,34625,1732481182291 in 420 msec 2024-11-24T20:46:25,460 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T20:46:25,460 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 652 msec 2024-11-24T20:46:25,462 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T20:46:25,463 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T20:46:25,489 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:46:25,490 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:46:25,515 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:46:25,518 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59535, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:46:25,543 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.3640 sec 2024-11-24T20:46:25,543 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732481185543, completionTime=-1 2024-11-24T20:46:25,546 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-24T20:46:25,547 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T20:46:25,581 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-24T20:46:25,581 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732481245581 2024-11-24T20:46:25,581 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732481305581 2024-11-24T20:46:25,581 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 34 msec 2024-11-24T20:46:25,584 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=20b981ef5c27,38339,1732481181363-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T20:46:25,585 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=20b981ef5c27,38339,1732481181363-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T20:46:25,585 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=20b981ef5c27,38339,1732481181363-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T20:46:25,587 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-20b981ef5c27:38339, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T20:46:25,587 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T20:46:25,588 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T20:46:25,598 DEBUG [master/20b981ef5c27:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T20:46:25,637 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.117sec 2024-11-24T20:46:25,638 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T20:46:25,640 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T20:46:25,641 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T20:46:25,642 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T20:46:25,642 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T20:46:25,643 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=20b981ef5c27,38339,1732481181363-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T20:46:25,644 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=20b981ef5c27,38339,1732481181363-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T20:46:25,654 DEBUG [master/20b981ef5c27:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T20:46:25,660 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T20:46:25,660 INFO [master/20b981ef5c27:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=20b981ef5c27,38339,1732481181363-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T20:46:25,736 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ec6ff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:25,738 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-24T20:46:25,738 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-24T20:46:25,741 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:46:25,743 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:46:25,750 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:46:25,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:46:25,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:46:25,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bb8961c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:25,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:46:25,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:46:25,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:46:25,758 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:46:25,761 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35f5c7f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:25,761 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:46:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:46:25,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:46:25,771 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40624, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:46:25,788 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=20b981ef5c27,38339,1732481181363 2024-11-24T20:46:25,799 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=473, ProcessCount=11, AvailableMemoryMB=10790 2024-11-24T20:46:25,807 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T20:46:25,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 20b981ef5c27,38339,1732481181363 2024-11-24T20:46:25,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6fcc49df 2024-11-24T20:46:25,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T20:46:25,818 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51612, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T20:46:25,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T20:46:25,825 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T20:46:25,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T20:46:25,836 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T20:46:25,838 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-24T20:46:25,839 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:46:25,842 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T20:46:25,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T20:46:25,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741835_1011 (size=959) 2024-11-24T20:46:25,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T20:46:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T20:46:26,259 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a 2024-11-24T20:46:26,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741836_1012 (size=53) 2024-11-24T20:46:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T20:46:26,673 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:46:26,674 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6ac72b5882ff154796d9a89886bf5150, disabling compactions & flushes 2024-11-24T20:46:26,674 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:26,674 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:26,675 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. after waiting 0 ms 2024-11-24T20:46:26,675 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:26,675 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:26,676 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6ac72b5882ff154796d9a89886bf5150: Waiting for close lock at 1732481186674Disabling compacts and flushes for region at 1732481186674Disabling writes for close at 1732481186675 (+1 ms)Writing region close event to WAL at 1732481186675Closed at 1732481186675 2024-11-24T20:46:26,679 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T20:46:26,684 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732481186679"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732481186679"}]},"ts":"1732481186679"} 2024-11-24T20:46:26,688 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T20:46:26,690 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T20:46:26,693 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732481186690"}]},"ts":"1732481186690"} 2024-11-24T20:46:26,697 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T20:46:26,699 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ac72b5882ff154796d9a89886bf5150, ASSIGN}] 2024-11-24T20:46:26,701 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ac72b5882ff154796d9a89886bf5150, ASSIGN 2024-11-24T20:46:26,703 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ac72b5882ff154796d9a89886bf5150, ASSIGN; state=OFFLINE, location=20b981ef5c27,34625,1732481182291; forceNewPlan=false, retain=false 2024-11-24T20:46:26,854 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6ac72b5882ff154796d9a89886bf5150, regionState=OPENING, regionLocation=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:26,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ac72b5882ff154796d9a89886bf5150, ASSIGN because future has completed 2024-11-24T20:46:26,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291}] 2024-11-24T20:46:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T20:46:27,020 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:27,020 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} 2024-11-24T20:46:27,021 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:27,021 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:46:27,021 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:27,022 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:27,025 INFO [StoreOpener-6ac72b5882ff154796d9a89886bf5150-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:27,028 INFO [StoreOpener-6ac72b5882ff154796d9a89886bf5150-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ac72b5882ff154796d9a89886bf5150 columnFamilyName A 2024-11-24T20:46:27,028 DEBUG [StoreOpener-6ac72b5882ff154796d9a89886bf5150-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:46:27,029 INFO [StoreOpener-6ac72b5882ff154796d9a89886bf5150-1 {}] regionserver.HStore(327): Store=6ac72b5882ff154796d9a89886bf5150/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:46:27,029 INFO [StoreOpener-6ac72b5882ff154796d9a89886bf5150-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:27,032 INFO [StoreOpener-6ac72b5882ff154796d9a89886bf5150-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ac72b5882ff154796d9a89886bf5150 columnFamilyName B 2024-11-24T20:46:27,032 DEBUG [StoreOpener-6ac72b5882ff154796d9a89886bf5150-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:46:27,033 INFO [StoreOpener-6ac72b5882ff154796d9a89886bf5150-1 {}] regionserver.HStore(327): Store=6ac72b5882ff154796d9a89886bf5150/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:46:27,033 INFO [StoreOpener-6ac72b5882ff154796d9a89886bf5150-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:27,036 INFO [StoreOpener-6ac72b5882ff154796d9a89886bf5150-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ac72b5882ff154796d9a89886bf5150 columnFamilyName C 2024-11-24T20:46:27,036 DEBUG [StoreOpener-6ac72b5882ff154796d9a89886bf5150-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:46:27,037 INFO [StoreOpener-6ac72b5882ff154796d9a89886bf5150-1 {}] regionserver.HStore(327): Store=6ac72b5882ff154796d9a89886bf5150/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:46:27,038 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:27,039 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:27,040 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:27,042 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:27,042 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:27,043 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T20:46:27,046 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:27,049 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T20:46:27,050 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 6ac72b5882ff154796d9a89886bf5150; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71209395, jitterRate=0.061102673411369324}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T20:46:27,050 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:27,051 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 6ac72b5882ff154796d9a89886bf5150: Running coprocessor pre-open hook at 1732481187022Writing region info on filesystem at 1732481187022Initializing all the Stores at 1732481187024 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481187024Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481187025 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481187025Cleaning up temporary data from old regions at 1732481187042 (+17 ms)Running coprocessor post-open hooks at 1732481187050 (+8 ms)Region opened successfully at 1732481187051 (+1 ms) 2024-11-24T20:46:27,053 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., pid=6, masterSystemTime=1732481187014 2024-11-24T20:46:27,056 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:27,056 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:27,057 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6ac72b5882ff154796d9a89886bf5150, regionState=OPEN, openSeqNum=2, regionLocation=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:27,061 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 because future has completed 2024-11-24T20:46:27,067 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T20:46:27,068 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 in 202 msec 2024-11-24T20:46:27,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T20:46:27,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ac72b5882ff154796d9a89886bf5150, ASSIGN in 368 msec 2024-11-24T20:46:27,073 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T20:46:27,073 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732481187073"}]},"ts":"1732481187073"} 2024-11-24T20:46:27,076 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T20:46:27,078 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T20:46:27,081 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2500 sec 2024-11-24T20:46:28,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T20:46:28,008 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:46:28,017 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@737445c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:28,017 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:46:28,018 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:46:28,019 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:46:28,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:46:28,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:46:28,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@253618fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:28,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:46:28,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:46:28,021 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:46:28,022 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51624, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:46:28,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67dec8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:28,024 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:46:28,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:46:28,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:46:28,028 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40630, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:46:28,031 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T20:46:28,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 20b981ef5c27,38339,1732481181363 2024-11-24T20:46:28,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1df1000 2024-11-24T20:46:28,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T20:46:28,037 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51636, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T20:46:28,041 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d7b17c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:28,041 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:46:28,041 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:46:28,043 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:46:28,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:46:28,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:46:28,043 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6b0c61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:46:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:46:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:46:28,046 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51644, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:46:28,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cdb8e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:28,048 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6068bcec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:28,048 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:46:28,048 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:46:28,049 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:46:28,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:46:28,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:46:28,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675bb8fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:28,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:46:28,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:46:28,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:46:28,053 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:46:28,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69b21050, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:28,055 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11fbf093, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:28,056 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:46:28,056 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:46:28,057 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:46:28,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:46:28,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:46:28,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a0960a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:28,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:46:28,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:46:28,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:46:28,061 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51682, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:46:28,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749fb54b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:28,063 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64f1c2cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:28,064 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:46:28,064 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:46:28,066 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:46:28,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:46:28,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:46:28,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68cf4e74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:28,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:46:28,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:46:28,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:46:28,070 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51702, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:46:28,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df7e3f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:28,073 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a0722e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:28,073 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:46:28,073 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:46:28,074 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:46:28,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:46:28,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:46:28,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41ae5b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:28,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:46:28,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:46:28,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:46:28,077 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51728, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:46:28,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@300379dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:28,081 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d6a39a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:28,081 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:46:28,081 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:46:28,082 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:46:28,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:46:28,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:46:28,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@102d4bb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:28,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:46:28,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:46:28,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:46:28,087 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51756, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:46:28,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ba092b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:28,089 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16b28440, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:28,089 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:46:28,089 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:46:28,091 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:46:28,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:46:28,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:46:28,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74deee7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:28,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:46:28,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:46:28,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:46:28,093 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51776, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:46:28,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e021fe2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:28,100 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b18ef51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:28,100 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:46:28,101 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:46:28,102 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:46:28,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:46:28,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:46:28,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e877dc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:28,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:46:28,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:46:28,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:46:28,105 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51792, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:46:28,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35351cdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:28,107 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c865cb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:28,108 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:46:28,108 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:46:28,109 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:46:28,109 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:46:28,110 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:46:28,110 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c39882a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:28,110 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:46:28,110 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:46:28,110 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:46:28,112 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:46:28,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c1ec368, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:46:28,116 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:46:28,116 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:46:28,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:46:28,118 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:46:28,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:46:28,118 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:46:28,119 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:46:28,120 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40638, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:46:28,120 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40636, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:46:28,122 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:46:28,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:46:28,124 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40652, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:46:28,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2] 2024-11-24T20:46:28,128 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:46:28,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:46:28,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:46:28,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2] 2024-11-24T20:46:28,133 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:46:28,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2] 2024-11-24T20:46:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2] 2024-11-24T20:46:28,138 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:46:28,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:46:28,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:46:28,142 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:46:28,142 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:46:28,144 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40668, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:46:28,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:46:28,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:46:28,149 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:46:28,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2] 2024-11-24T20:46:28,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:46:28,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:46:28,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:46:28,153 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40672, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:46:28,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:46:28,155 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40684, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:46:28,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2] 2024-11-24T20:46:28,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2] 2024-11-24T20:46:28,161 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40682, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:46:28,166 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:46:28,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-24T20:46:28,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2] 2024-11-24T20:46:28,180 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:46:28,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T20:46:28,180 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:46:28,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:46:28,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:46:28,187 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40700, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:46:28,192 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:46:28,194 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:46:28,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2] 2024-11-24T20:46:28,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:28,268 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:46:28,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ac72b5882ff154796d9a89886bf5150 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T20:46:28,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T20:46:28,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T20:46:28,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:28,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:28,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:28,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:28,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:28,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:28,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/b33b7c5cb4ac41c8ad50f6a380a1f86c is 50, key is test_row_0/A:col10/1732481188247/Put/seqid=0 2024-11-24T20:46:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T20:46:28,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481248491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:28,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741837_1013 (size=12001) 2024-11-24T20:46:28,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/b33b7c5cb4ac41c8ad50f6a380a1f86c 2024-11-24T20:46:28,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481248502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:28,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481248502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:28,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:28,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:28,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481248507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:28,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481248512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:28,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:28,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:28,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:28,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:28,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:28,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:28,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:28,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:28,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:28,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:28,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:28,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:28,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:28,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:28,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:28,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T20:46:28,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:28,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:28,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:28,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:28,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:28,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:28,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481248656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:28,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:28,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:28,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:28,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/a474b9ff974f404089203e11854c9430 is 50, key is test_row_0/B:col10/1732481188247/Put/seqid=0 2024-11-24T20:46:28,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481248657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:28,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481248656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:28,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:28,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:28,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:28,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481248658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:28,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:28,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:28,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:28,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481248657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:28,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:28,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:28,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:28,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:28,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:28,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:28,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741838_1014 (size=12001) 2024-11-24T20:46:28,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T20:46:28,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:28,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:28,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:28,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:28,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:28,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T20:46:28,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:28,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481248887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:28,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:28,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:28,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:28,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:28,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481248887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:28,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:28,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:28,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:28,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:28,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481248888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:28,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:28,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:28,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:28,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:28,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:28,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481248890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:28,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481248890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:28,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:28,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:28,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:28,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:28,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:28,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:28,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T20:46:28,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:28,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:28,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:28,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:28,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:28,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:29,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T20:46:29,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:29,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:29,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:29,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:29,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:29,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:29,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/a474b9ff974f404089203e11854c9430 2024-11-24T20:46:29,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/b37fb7b205ef48ddbfa28b9354fd0c8f is 50, key is test_row_0/C:col10/1732481188247/Put/seqid=0 2024-11-24T20:46:29,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:29,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:29,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481249206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481249207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481249207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:29,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:29,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:29,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:29,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:29,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:29,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:29,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:29,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:29,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:29,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:29,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481249208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481249208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:29,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:29,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:29,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:29,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:29,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:29,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:29,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T20:46:29,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:29,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:29,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:29,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:29,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:29,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741839_1015 (size=12001) 2024-11-24T20:46:29,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/b37fb7b205ef48ddbfa28b9354fd0c8f 2024-11-24T20:46:29,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/b33b7c5cb4ac41c8ad50f6a380a1f86c as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/b33b7c5cb4ac41c8ad50f6a380a1f86c 2024-11-24T20:46:29,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/b33b7c5cb4ac41c8ad50f6a380a1f86c, entries=150, sequenceid=13, filesize=11.7 K 2024-11-24T20:46:29,302 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-24T20:46:29,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/a474b9ff974f404089203e11854c9430 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/a474b9ff974f404089203e11854c9430 2024-11-24T20:46:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T20:46:29,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/a474b9ff974f404089203e11854c9430, entries=150, sequenceid=13, filesize=11.7 K 2024-11-24T20:46:29,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/b37fb7b205ef48ddbfa28b9354fd0c8f as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/b37fb7b205ef48ddbfa28b9354fd0c8f 2024-11-24T20:46:29,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/b37fb7b205ef48ddbfa28b9354fd0c8f, entries=150, sequenceid=13, filesize=11.7 K 2024-11-24T20:46:29,370 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 6ac72b5882ff154796d9a89886bf5150 in 1091ms, sequenceid=13, compaction requested=false 2024-11-24T20:46:29,371 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-24T20:46:29,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:29,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T20:46:29,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:29,398 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 6ac72b5882ff154796d9a89886bf5150 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-24T20:46:29,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/76cdb06b45674600959e55dcef63a574 is 50, key is test_row_0/A:col10/1732481188486/Put/seqid=0 2024-11-24T20:46:29,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741840_1016 (size=12001) 2024-11-24T20:46:29,436 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/76cdb06b45674600959e55dcef63a574 2024-11-24T20:46:29,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/223cc249e095465ea97c498104f9a0aa is 50, key is test_row_0/B:col10/1732481188486/Put/seqid=0 2024-11-24T20:46:29,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741841_1017 (size=12001) 2024-11-24T20:46:29,482 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/223cc249e095465ea97c498104f9a0aa 2024-11-24T20:46:29,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/63857b9dff51435bb247a71277f01c76 is 50, key is test_row_0/C:col10/1732481188486/Put/seqid=0 2024-11-24T20:46:29,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741842_1018 (size=12001) 2024-11-24T20:46:29,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:29,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:29,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:29,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481249773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:29,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:29,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:29,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:29,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:29,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481249776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:29,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:29,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:29,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:29,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:29,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481249794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:29,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:29,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:29,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:29,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:29,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:29,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481249799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:29,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:29,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:29,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481249802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:29,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:29,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:29,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:29,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:29,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:29,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481249907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:29,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:29,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:29,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:29,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:29,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481249915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:29,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:29,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:29,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:29,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:29,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481249927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:29,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:29,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:29,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:29,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:29,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481249927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:29,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:29,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:29,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:29,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:29,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481249928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:29,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:29,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:29,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:29,984 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/63857b9dff51435bb247a71277f01c76 2024-11-24T20:46:30,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/76cdb06b45674600959e55dcef63a574 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/76cdb06b45674600959e55dcef63a574 2024-11-24T20:46:30,024 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/76cdb06b45674600959e55dcef63a574, entries=150, sequenceid=38, filesize=11.7 K 2024-11-24T20:46:30,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/223cc249e095465ea97c498104f9a0aa as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/223cc249e095465ea97c498104f9a0aa 2024-11-24T20:46:30,043 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/223cc249e095465ea97c498104f9a0aa, entries=150, sequenceid=38, filesize=11.7 K 2024-11-24T20:46:30,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/63857b9dff51435bb247a71277f01c76 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/63857b9dff51435bb247a71277f01c76 2024-11-24T20:46:30,072 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/63857b9dff51435bb247a71277f01c76, entries=150, sequenceid=38, filesize=11.7 K 2024-11-24T20:46:30,075 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 6ac72b5882ff154796d9a89886bf5150 in 678ms, sequenceid=38, compaction requested=false 2024-11-24T20:46:30,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:30,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:30,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-24T20:46:30,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-24T20:46:30,085 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T20:46:30,085 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8860 sec 2024-11-24T20:46:30,091 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.9180 sec 2024-11-24T20:46:30,122 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:46:30,122 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ac72b5882ff154796d9a89886bf5150 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-24T20:46:30,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:30,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/0bcd450ce37e4d3891580d8fd09818dc is 50, key is test_row_0/A:col10/1732481189798/Put/seqid=0 2024-11-24T20:46:30,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741843_1019 (size=11997) 2024-11-24T20:46:30,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/0bcd450ce37e4d3891580d8fd09818dc 2024-11-24T20:46:30,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/051d5b8f354c474b942c7ba563da946e is 50, key is test_row_0/B:col10/1732481189798/Put/seqid=0 2024-11-24T20:46:30,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741844_1020 (size=9657) 2024-11-24T20:46:30,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/051d5b8f354c474b942c7ba563da946e 2024-11-24T20:46:30,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:30,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481250278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:30,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:30,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:30,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481250288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:30,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:30,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:30,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481250279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:30,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:30,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481250297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:30,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:30,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:30,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481250313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:30,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:30,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:30,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T20:46:30,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/57de0894180e49528f1cb5cbb0b18d8e is 50, key is test_row_0/C:col10/1732481189798/Put/seqid=0 2024-11-24T20:46:30,336 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:46:30,337 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:46:30,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-24T20:46:30,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T20:46:30,341 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:46:30,344 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:46:30,344 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:46:30,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741845_1021 (size=9657) 2024-11-24T20:46:30,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/57de0894180e49528f1cb5cbb0b18d8e 2024-11-24T20:46:30,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/0bcd450ce37e4d3891580d8fd09818dc as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/0bcd450ce37e4d3891580d8fd09818dc 2024-11-24T20:46:30,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/0bcd450ce37e4d3891580d8fd09818dc, entries=150, sequenceid=50, filesize=11.7 K 2024-11-24T20:46:30,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/051d5b8f354c474b942c7ba563da946e as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/051d5b8f354c474b942c7ba563da946e 2024-11-24T20:46:30,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:30,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:30,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481250426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:30,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481250426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:30,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:30,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:30,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:30,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:30,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481250427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:30,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481250427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:30,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/051d5b8f354c474b942c7ba563da946e, entries=100, sequenceid=50, filesize=9.4 K 2024-11-24T20:46:30,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:30,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:30,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:30,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481250427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:30,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:30,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/57de0894180e49528f1cb5cbb0b18d8e as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/57de0894180e49528f1cb5cbb0b18d8e 2024-11-24T20:46:30,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T20:46:30,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/57de0894180e49528f1cb5cbb0b18d8e, entries=100, sequenceid=50, filesize=9.4 K 2024-11-24T20:46:30,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 6ac72b5882ff154796d9a89886bf5150 in 333ms, sequenceid=50, compaction requested=true 2024-11-24T20:46:30,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:30,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:46:30,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:46:30,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:46:30,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:46:30,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:46:30,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:46:30,477 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:46:30,477 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:46:30,482 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:46:30,484 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 6ac72b5882ff154796d9a89886bf5150/B is initiating minor compaction (all files) 2024-11-24T20:46:30,484 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ac72b5882ff154796d9a89886bf5150/B in TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:30,485 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/a474b9ff974f404089203e11854c9430, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/223cc249e095465ea97c498104f9a0aa, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/051d5b8f354c474b942c7ba563da946e] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp, totalSize=32.9 K 2024-11-24T20:46:30,486 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:46:30,486 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 6ac72b5882ff154796d9a89886bf5150/A is initiating minor compaction (all files) 2024-11-24T20:46:30,486 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ac72b5882ff154796d9a89886bf5150/A in TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:30,487 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/b33b7c5cb4ac41c8ad50f6a380a1f86c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/76cdb06b45674600959e55dcef63a574, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/0bcd450ce37e4d3891580d8fd09818dc] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp, totalSize=35.2 K 2024-11-24T20:46:30,488 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting a474b9ff974f404089203e11854c9430, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732481188228 2024-11-24T20:46:30,488 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting b33b7c5cb4ac41c8ad50f6a380a1f86c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732481188228 2024-11-24T20:46:30,489 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 223cc249e095465ea97c498104f9a0aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732481188483 2024-11-24T20:46:30,490 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 051d5b8f354c474b942c7ba563da946e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732481189798 2024-11-24T20:46:30,495 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76cdb06b45674600959e55dcef63a574, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732481188483 2024-11-24T20:46:30,500 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bcd450ce37e4d3891580d8fd09818dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732481189778 2024-11-24T20:46:30,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T20:46:30,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:30,508 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 6ac72b5882ff154796d9a89886bf5150 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-24T20:46:30,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/f5a6e4290843488da599a213cd7f060b is 50, key is test_row_0/A:col10/1732481190274/Put/seqid=0 2024-11-24T20:46:30,566 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ac72b5882ff154796d9a89886bf5150#A#compaction#11 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:46:30,566 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ac72b5882ff154796d9a89886bf5150#B#compaction#10 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:46:30,572 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/6c117859f86f4515b1f31496a9f1e0d7 is 50, key is test_row_0/A:col10/1732481189798/Put/seqid=0 2024-11-24T20:46:30,572 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/2bba43e13a494a768f4aa87b6cfd1512 is 50, key is test_row_0/B:col10/1732481189798/Put/seqid=0 2024-11-24T20:46:30,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741846_1022 (size=12001) 2024-11-24T20:46:30,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741848_1024 (size=12104) 2024-11-24T20:46:30,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741847_1023 (size=12104) 2024-11-24T20:46:30,636 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/6c117859f86f4515b1f31496a9f1e0d7 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/6c117859f86f4515b1f31496a9f1e0d7 2024-11-24T20:46:30,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:30,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:30,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T20:46:30,657 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/2bba43e13a494a768f4aa87b6cfd1512 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/2bba43e13a494a768f4aa87b6cfd1512 2024-11-24T20:46:30,683 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ac72b5882ff154796d9a89886bf5150/B of 6ac72b5882ff154796d9a89886bf5150 into 2bba43e13a494a768f4aa87b6cfd1512(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:46:30,683 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:30,686 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ac72b5882ff154796d9a89886bf5150/A of 6ac72b5882ff154796d9a89886bf5150 into 6c117859f86f4515b1f31496a9f1e0d7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:46:30,686 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:30,688 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., storeName=6ac72b5882ff154796d9a89886bf5150/A, priority=13, startTime=1732481190457; duration=0sec 2024-11-24T20:46:30,689 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:46:30,689 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:A 2024-11-24T20:46:30,689 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:46:30,690 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., storeName=6ac72b5882ff154796d9a89886bf5150/B, priority=13, startTime=1732481190475; duration=0sec 2024-11-24T20:46:30,691 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:46:30,691 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:B 2024-11-24T20:46:30,700 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:46:30,700 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 6ac72b5882ff154796d9a89886bf5150/C is initiating minor compaction (all files) 2024-11-24T20:46:30,700 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ac72b5882ff154796d9a89886bf5150/C in TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:30,700 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/b37fb7b205ef48ddbfa28b9354fd0c8f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/63857b9dff51435bb247a71277f01c76, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/57de0894180e49528f1cb5cbb0b18d8e] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp, totalSize=32.9 K 2024-11-24T20:46:30,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:30,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481250679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:30,702 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting b37fb7b205ef48ddbfa28b9354fd0c8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732481188228 2024-11-24T20:46:30,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:30,703 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63857b9dff51435bb247a71277f01c76, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732481188483 2024-11-24T20:46:30,704 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57de0894180e49528f1cb5cbb0b18d8e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732481189798 2024-11-24T20:46:30,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:30,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481250694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:30,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:30,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481250695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:30,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:30,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:30,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481250700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:30,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:30,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:30,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481250702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:30,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:30,752 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ac72b5882ff154796d9a89886bf5150#C#compaction#12 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:46:30,753 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/5386731971ce4af99126a8c15227240b is 50, key is test_row_0/C:col10/1732481189798/Put/seqid=0 2024-11-24T20:46:30,773 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T20:46:30,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741849_1025 (size=12104) 2024-11-24T20:46:30,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:30,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481250805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:30,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:30,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:30,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481250826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:30,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:30,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:30,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481250826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:30,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:30,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:30,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481250826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:30,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:30,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:30,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481250836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:30,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:30,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:30,910 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T20:46:30,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T20:46:30,977 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/f5a6e4290843488da599a213cd7f060b 2024-11-24T20:46:31,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/5e536cc8b9914d029fbe2f1586a5d9e6 is 50, key is test_row_0/B:col10/1732481190274/Put/seqid=0 2024-11-24T20:46:31,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481251035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:31,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:31,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:31,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:31,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481251046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:31,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:31,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:31,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:31,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741850_1026 (size=12001) 2024-11-24T20:46:31,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481251056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:31,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:31,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:31,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481251056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481251056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:31,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:31,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:31,232 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/5386731971ce4af99126a8c15227240b as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/5386731971ce4af99126a8c15227240b 2024-11-24T20:46:31,250 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ac72b5882ff154796d9a89886bf5150/C of 6ac72b5882ff154796d9a89886bf5150 into 5386731971ce4af99126a8c15227240b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:46:31,251 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:31,251 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., storeName=6ac72b5882ff154796d9a89886bf5150/C, priority=13, startTime=1732481190475; duration=0sec 2024-11-24T20:46:31,251 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:46:31,251 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:C 2024-11-24T20:46:31,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:31,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481251346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:31,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:31,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:31,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:31,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:31,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481251355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:31,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:31,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:31,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:31,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:31,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481251366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:31,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481251367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:31,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:31,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:31,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:31,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:31,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481251367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:31,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:31,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:31,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:31,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:31,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:31,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:31,454 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/5e536cc8b9914d029fbe2f1586a5d9e6 2024-11-24T20:46:31,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/4bb711251c204ff79d2d1915027a8e16 is 50, key is test_row_0/C:col10/1732481190274/Put/seqid=0 2024-11-24T20:46:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T20:46:31,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741851_1027 (size=12001) 2024-11-24T20:46:31,494 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/4bb711251c204ff79d2d1915027a8e16 2024-11-24T20:46:31,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/f5a6e4290843488da599a213cd7f060b as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/f5a6e4290843488da599a213cd7f060b 2024-11-24T20:46:31,525 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/f5a6e4290843488da599a213cd7f060b, entries=150, sequenceid=74, filesize=11.7 K 2024-11-24T20:46:31,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/5e536cc8b9914d029fbe2f1586a5d9e6 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/5e536cc8b9914d029fbe2f1586a5d9e6 2024-11-24T20:46:31,543 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/5e536cc8b9914d029fbe2f1586a5d9e6, entries=150, sequenceid=74, filesize=11.7 K 2024-11-24T20:46:31,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/4bb711251c204ff79d2d1915027a8e16 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/4bb711251c204ff79d2d1915027a8e16 2024-11-24T20:46:31,567 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/4bb711251c204ff79d2d1915027a8e16, entries=150, sequenceid=74, filesize=11.7 K 2024-11-24T20:46:31,569 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 6ac72b5882ff154796d9a89886bf5150 in 1061ms, sequenceid=74, compaction requested=false 2024-11-24T20:46:31,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:31,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:31,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-24T20:46:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-24T20:46:31,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-24T20:46:31,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2280 sec 2024-11-24T20:46:31,589 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.2460 sec 2024-11-24T20:46:31,800 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-24T20:46:31,801 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-24T20:46:31,803 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T20:46:31,803 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-24T20:46:31,804 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-24T20:46:31,804 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-24T20:46:31,806 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-24T20:46:31,806 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-24T20:46:31,862 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:46:31,862 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ac72b5882ff154796d9a89886bf5150 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-24T20:46:31,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:31,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/56b0ff1375354160be9d13c28ced7bc6 is 50, key is test_row_0/A:col10/1732481190695/Put/seqid=0 2024-11-24T20:46:31,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741852_1028 (size=14341) 2024-11-24T20:46:31,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/56b0ff1375354160be9d13c28ced7bc6 2024-11-24T20:46:31,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/ee3bc25253a04a1a8c34200feaeaff36 is 50, key is test_row_0/B:col10/1732481190695/Put/seqid=0 2024-11-24T20:46:31,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741853_1029 (size=12001) 2024-11-24T20:46:31,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:31,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481251979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:31,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:31,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:31,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:31,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:31,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481251981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:31,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:31,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:31,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:31,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:31,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481251984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:31,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:31,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:31,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:31,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:31,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481251989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:31,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:31,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:31,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:31,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:31,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481251993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:32,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:32,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:32,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481252096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:32,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:32,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:32,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481252096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:32,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:32,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481252096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:32,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:32,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:32,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:32,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481252105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:32,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:32,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:32,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481252105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:32,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:32,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:32,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481252305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:32,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:32,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:32,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:32,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481252305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:32,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481252305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:32,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:32,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:32,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:32,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481252315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:32,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:32,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:32,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481252315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:32,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:32,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/ee3bc25253a04a1a8c34200feaeaff36 2024-11-24T20:46:32,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/2e333cb0c2a5457ab3448deeade74977 is 50, key is test_row_0/C:col10/1732481190695/Put/seqid=0 2024-11-24T20:46:32,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741854_1030 (size=12001) 2024-11-24T20:46:32,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/2e333cb0c2a5457ab3448deeade74977 2024-11-24T20:46:32,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/56b0ff1375354160be9d13c28ced7bc6 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/56b0ff1375354160be9d13c28ced7bc6 2024-11-24T20:46:32,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/56b0ff1375354160be9d13c28ced7bc6, entries=200, sequenceid=91, filesize=14.0 K 2024-11-24T20:46:32,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/ee3bc25253a04a1a8c34200feaeaff36 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/ee3bc25253a04a1a8c34200feaeaff36 2024-11-24T20:46:32,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/ee3bc25253a04a1a8c34200feaeaff36, entries=150, sequenceid=91, filesize=11.7 K 2024-11-24T20:46:32,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/2e333cb0c2a5457ab3448deeade74977 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/2e333cb0c2a5457ab3448deeade74977 2024-11-24T20:46:32,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T20:46:32,486 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:46:32,487 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:46:32,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-24T20:46:32,493 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:46:32,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T20:46:32,495 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:46:32,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:46:32,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/2e333cb0c2a5457ab3448deeade74977, entries=150, sequenceid=91, filesize=11.7 K 2024-11-24T20:46:32,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 6ac72b5882ff154796d9a89886bf5150 in 638ms, sequenceid=91, compaction requested=true 2024-11-24T20:46:32,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:32,501 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:46:32,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:46:32,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:46:32,501 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:46:32,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:46:32,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:46:32,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:46:32,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:46:32,504 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:46:32,504 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 6ac72b5882ff154796d9a89886bf5150/B is initiating minor compaction (all files) 2024-11-24T20:46:32,504 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ac72b5882ff154796d9a89886bf5150/B in TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:32,504 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/2bba43e13a494a768f4aa87b6cfd1512, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/5e536cc8b9914d029fbe2f1586a5d9e6, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/ee3bc25253a04a1a8c34200feaeaff36] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp, totalSize=35.3 K 2024-11-24T20:46:32,505 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:46:32,505 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 6ac72b5882ff154796d9a89886bf5150/A is initiating minor compaction (all files) 2024-11-24T20:46:32,505 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ac72b5882ff154796d9a89886bf5150/A in TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:32,506 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/6c117859f86f4515b1f31496a9f1e0d7, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/f5a6e4290843488da599a213cd7f060b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/56b0ff1375354160be9d13c28ced7bc6] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp, totalSize=37.5 K 2024-11-24T20:46:32,506 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bba43e13a494a768f4aa87b6cfd1512, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732481188483 2024-11-24T20:46:32,507 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c117859f86f4515b1f31496a9f1e0d7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732481188483 2024-11-24T20:46:32,507 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e536cc8b9914d029fbe2f1586a5d9e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732481190274 2024-11-24T20:46:32,507 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5a6e4290843488da599a213cd7f060b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732481190274 2024-11-24T20:46:32,508 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting ee3bc25253a04a1a8c34200feaeaff36, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732481190665 2024-11-24T20:46:32,508 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56b0ff1375354160be9d13c28ced7bc6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732481190665 2024-11-24T20:46:32,528 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ac72b5882ff154796d9a89886bf5150#A#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:46:32,529 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/546476d909514a0191fe37f45f5556e9 is 50, key is test_row_0/A:col10/1732481190695/Put/seqid=0 2024-11-24T20:46:32,533 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ac72b5882ff154796d9a89886bf5150#B#compaction#19 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:46:32,534 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/ab6224f550a643a9a44bc03dd2b47a24 is 50, key is test_row_0/B:col10/1732481190695/Put/seqid=0 2024-11-24T20:46:32,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741855_1031 (size=12207) 2024-11-24T20:46:32,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T20:46:32,625 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:46:32,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741856_1032 (size=12207) 2024-11-24T20:46:32,625 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ac72b5882ff154796d9a89886bf5150 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-24T20:46:32,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:32,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:32,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481252642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:32,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:32,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:32,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481252642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:32,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:32,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T20:46:32,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:32,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:32,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:32,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:32,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:32,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/4113f64b8d854c30b5e89dc166d23c03 is 50, key is test_row_0/A:col10/1732481192622/Put/seqid=0 2024-11-24T20:46:32,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:32,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:32,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481252646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:32,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:32,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:32,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481252647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:32,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:32,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481252649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:32,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:32,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:32,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741857_1033 (size=14341) 2024-11-24T20:46:32,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/4113f64b8d854c30b5e89dc166d23c03 2024-11-24T20:46:32,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/2a229fcffe6043a789c57a6e1df07697 is 50, key is test_row_0/B:col10/1732481192622/Put/seqid=0 2024-11-24T20:46:32,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741858_1034 (size=12001) 2024-11-24T20:46:32,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/2a229fcffe6043a789c57a6e1df07697 2024-11-24T20:46:32,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:32,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481252755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:32,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:32,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481252755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:32,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/3a7a1067db1f4d82bc4f144c743a9142 is 50, key is test_row_0/C:col10/1732481192622/Put/seqid=0 2024-11-24T20:46:32,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:32,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:32,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:32,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481252766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:32,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:32,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481252766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:32,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:32,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:32,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:32,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481252766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:32,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:32,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:32,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741859_1035 (size=12001) 2024-11-24T20:46:32,801 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/3a7a1067db1f4d82bc4f144c743a9142 2024-11-24T20:46:32,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T20:46:32,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:32,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:32,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:32,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:32,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:32,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:32,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/4113f64b8d854c30b5e89dc166d23c03 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/4113f64b8d854c30b5e89dc166d23c03 2024-11-24T20:46:32,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T20:46:32,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/4113f64b8d854c30b5e89dc166d23c03, entries=200, sequenceid=117, filesize=14.0 K 2024-11-24T20:46:32,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/2a229fcffe6043a789c57a6e1df07697 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/2a229fcffe6043a789c57a6e1df07697 2024-11-24T20:46:32,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/2a229fcffe6043a789c57a6e1df07697, entries=150, sequenceid=117, filesize=11.7 K 2024-11-24T20:46:32,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/3a7a1067db1f4d82bc4f144c743a9142 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/3a7a1067db1f4d82bc4f144c743a9142 2024-11-24T20:46:32,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/3a7a1067db1f4d82bc4f144c743a9142, entries=150, sequenceid=117, filesize=11.7 K 2024-11-24T20:46:32,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 6ac72b5882ff154796d9a89886bf5150 in 243ms, sequenceid=117, compaction requested=true 2024-11-24T20:46:32,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:32,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:A, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:46:32,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T20:46:32,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:B, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:46:32,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-24T20:46:32,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:46:32,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-24T20:46:32,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T20:46:32,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:32,964 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 6ac72b5882ff154796d9a89886bf5150 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T20:46:32,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/e35c760e29394812b5c68d17da6b4dad is 50, key is test_row_0/A:col10/1732481192645/Put/seqid=0 2024-11-24T20:46:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:32,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:33,011 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/546476d909514a0191fe37f45f5556e9 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/546476d909514a0191fe37f45f5556e9 2024-11-24T20:46:33,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741860_1036 (size=12001) 2024-11-24T20:46:33,037 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ac72b5882ff154796d9a89886bf5150/A of 6ac72b5882ff154796d9a89886bf5150 into 546476d909514a0191fe37f45f5556e9(size=11.9 K), total size for store is 25.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:46:33,037 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:33,037 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., storeName=6ac72b5882ff154796d9a89886bf5150/A, priority=13, startTime=1732481192501; duration=0sec 2024-11-24T20:46:33,038 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-24T20:46:33,038 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:A 2024-11-24T20:46:33,038 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:A 2024-11-24T20:46:33,038 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:46:33,038 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/e35c760e29394812b5c68d17da6b4dad 2024-11-24T20:46:33,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:33,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481253027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:33,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:33,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:33,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481253033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:33,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:33,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:33,050 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/ab6224f550a643a9a44bc03dd2b47a24 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/ab6224f550a643a9a44bc03dd2b47a24 2024-11-24T20:46:33,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481253036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:33,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:33,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:33,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481253040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:33,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:33,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:33,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481253040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:33,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:33,062 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:46:33,062 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 6ac72b5882ff154796d9a89886bf5150/C is initiating minor compaction (all files) 2024-11-24T20:46:33,063 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ac72b5882ff154796d9a89886bf5150/C in TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:33,063 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/5386731971ce4af99126a8c15227240b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/4bb711251c204ff79d2d1915027a8e16, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/2e333cb0c2a5457ab3448deeade74977, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/3a7a1067db1f4d82bc4f144c743a9142] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp, totalSize=47.0 K 2024-11-24T20:46:33,071 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ac72b5882ff154796d9a89886bf5150/B of 6ac72b5882ff154796d9a89886bf5150 into ab6224f550a643a9a44bc03dd2b47a24(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:46:33,071 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:33,071 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., storeName=6ac72b5882ff154796d9a89886bf5150/B, priority=13, startTime=1732481192501; duration=0sec 2024-11-24T20:46:33,071 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-24T20:46:33,071 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:B 2024-11-24T20:46:33,071 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:B 2024-11-24T20:46:33,071 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-24T20:46:33,071 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-24T20:46:33,072 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-24T20:46:33,072 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. because compaction request was cancelled 2024-11-24T20:46:33,072 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:C 2024-11-24T20:46:33,072 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-24T20:46:33,073 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5386731971ce4af99126a8c15227240b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732481188483 2024-11-24T20:46:33,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/b71cc8f2c1ae4ba3b055094863b94bae is 50, key is test_row_0/B:col10/1732481192645/Put/seqid=0 2024-11-24T20:46:33,077 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-24T20:46:33,077 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-24T20:46:33,078 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. because compaction request was cancelled 2024-11-24T20:46:33,078 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:B 2024-11-24T20:46:33,078 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-24T20:46:33,078 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bb711251c204ff79d2d1915027a8e16, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732481190274 2024-11-24T20:46:33,079 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-24T20:46:33,080 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-24T20:46:33,080 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. because compaction request was cancelled 2024-11-24T20:46:33,080 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:A 2024-11-24T20:46:33,087 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e333cb0c2a5457ab3448deeade74977, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732481190665 2024-11-24T20:46:33,088 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a7a1067db1f4d82bc4f144c743a9142, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732481191977 2024-11-24T20:46:33,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741861_1037 (size=12001) 2024-11-24T20:46:33,121 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/b71cc8f2c1ae4ba3b055094863b94bae 2024-11-24T20:46:33,123 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ac72b5882ff154796d9a89886bf5150#C#compaction#25 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:46:33,125 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/030d3bc82f5344aa86b3189f5a62fc7d is 50, key is test_row_0/C:col10/1732481192622/Put/seqid=0 2024-11-24T20:46:33,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T20:46:33,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:33,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481253147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:33,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:33,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:33,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481253155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:33,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:33,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:33,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/98310e8844ce4638bc14629709b63152 is 50, key is test_row_0/C:col10/1732481192645/Put/seqid=0 2024-11-24T20:46:33,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481253155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:33,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:33,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481253155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:33,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:33,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:33,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:33,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481253166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:33,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:33,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741862_1038 (size=12241) 2024-11-24T20:46:33,207 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/030d3bc82f5344aa86b3189f5a62fc7d as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/030d3bc82f5344aa86b3189f5a62fc7d 2024-11-24T20:46:33,220 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ac72b5882ff154796d9a89886bf5150/C of 6ac72b5882ff154796d9a89886bf5150 into 030d3bc82f5344aa86b3189f5a62fc7d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:46:33,220 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:33,222 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., storeName=6ac72b5882ff154796d9a89886bf5150/C, priority=12, startTime=1732481192869; duration=0sec 2024-11-24T20:46:33,222 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:46:33,222 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:C 2024-11-24T20:46:33,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741863_1039 (size=12001) 2024-11-24T20:46:33,223 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/98310e8844ce4638bc14629709b63152 2024-11-24T20:46:33,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/e35c760e29394812b5c68d17da6b4dad as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/e35c760e29394812b5c68d17da6b4dad 2024-11-24T20:46:33,254 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/e35c760e29394812b5c68d17da6b4dad, entries=150, sequenceid=128, filesize=11.7 K 2024-11-24T20:46:33,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/b71cc8f2c1ae4ba3b055094863b94bae as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/b71cc8f2c1ae4ba3b055094863b94bae 2024-11-24T20:46:33,274 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/b71cc8f2c1ae4ba3b055094863b94bae, entries=150, sequenceid=128, filesize=11.7 K 2024-11-24T20:46:33,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/98310e8844ce4638bc14629709b63152 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/98310e8844ce4638bc14629709b63152 2024-11-24T20:46:33,293 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/98310e8844ce4638bc14629709b63152, entries=150, sequenceid=128, filesize=11.7 K 2024-11-24T20:46:33,296 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 6ac72b5882ff154796d9a89886bf5150 in 333ms, sequenceid=128, compaction requested=true 2024-11-24T20:46:33,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:33,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:33,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-24T20:46:33,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-24T20:46:33,304 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-24T20:46:33,304 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 804 msec 2024-11-24T20:46:33,309 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 819 msec 2024-11-24T20:46:33,364 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:46:33,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ac72b5882ff154796d9a89886bf5150 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-24T20:46:33,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:33,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/687ef162984d4a10bbc54d6acd5bdc19 is 50, key is test_row_0/A:col10/1732481193031/Put/seqid=0 2024-11-24T20:46:33,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:33,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481253381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:33,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481253386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:33,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:33,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:33,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:33,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481253396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:33,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:33,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:33,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481253396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:33,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:33,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481253394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:33,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:33,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741864_1040 (size=12151) 2024-11-24T20:46:33,440 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/687ef162984d4a10bbc54d6acd5bdc19 2024-11-24T20:46:33,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/d5ba3f1169364ee9a247aa21796c7abf is 50, key is test_row_0/B:col10/1732481193031/Put/seqid=0 2024-11-24T20:46:33,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741865_1041 (size=12151) 2024-11-24T20:46:33,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/d5ba3f1169364ee9a247aa21796c7abf 2024-11-24T20:46:33,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:33,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481253506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:33,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:33,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/9e613f26f0fa48b7be95635a01ffbf68 is 50, key is test_row_0/C:col10/1732481193031/Put/seqid=0 2024-11-24T20:46:33,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:33,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481253516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:33,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:33,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:33,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481253516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:33,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:33,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:33,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481253525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:33,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:33,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741866_1042 (size=12151) 2024-11-24T20:46:33,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/9e613f26f0fa48b7be95635a01ffbf68 2024-11-24T20:46:33,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/687ef162984d4a10bbc54d6acd5bdc19 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/687ef162984d4a10bbc54d6acd5bdc19 2024-11-24T20:46:33,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/687ef162984d4a10bbc54d6acd5bdc19, entries=150, sequenceid=157, filesize=11.9 K 2024-11-24T20:46:33,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/d5ba3f1169364ee9a247aa21796c7abf as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/d5ba3f1169364ee9a247aa21796c7abf 2024-11-24T20:46:33,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/d5ba3f1169364ee9a247aa21796c7abf, entries=150, sequenceid=157, filesize=11.9 K 2024-11-24T20:46:33,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/9e613f26f0fa48b7be95635a01ffbf68 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/9e613f26f0fa48b7be95635a01ffbf68 2024-11-24T20:46:33,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/9e613f26f0fa48b7be95635a01ffbf68, entries=150, sequenceid=157, filesize=11.9 K 2024-11-24T20:46:33,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 6ac72b5882ff154796d9a89886bf5150 in 243ms, sequenceid=157, compaction requested=true 2024-11-24T20:46:33,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:33,608 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:46:33,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:46:33,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:46:33,608 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:46:33,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:46:33,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:46:33,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:46:33,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:46:33,612 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:46:33,612 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50700 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:46:33,613 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 6ac72b5882ff154796d9a89886bf5150/B is initiating minor compaction (all files) 2024-11-24T20:46:33,613 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 6ac72b5882ff154796d9a89886bf5150/A is initiating minor compaction (all files) 2024-11-24T20:46:33,613 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ac72b5882ff154796d9a89886bf5150/B in TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:33,613 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ac72b5882ff154796d9a89886bf5150/A in TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:33,613 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/ab6224f550a643a9a44bc03dd2b47a24, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/2a229fcffe6043a789c57a6e1df07697, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/b71cc8f2c1ae4ba3b055094863b94bae, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/d5ba3f1169364ee9a247aa21796c7abf] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp, totalSize=47.2 K 2024-11-24T20:46:33,613 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/546476d909514a0191fe37f45f5556e9, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/4113f64b8d854c30b5e89dc166d23c03, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/e35c760e29394812b5c68d17da6b4dad, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/687ef162984d4a10bbc54d6acd5bdc19] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp, totalSize=49.5 K 2024-11-24T20:46:33,614 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting ab6224f550a643a9a44bc03dd2b47a24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732481190665 2024-11-24T20:46:33,614 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 546476d909514a0191fe37f45f5556e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732481190665 2024-11-24T20:46:33,615 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4113f64b8d854c30b5e89dc166d23c03, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732481191977 2024-11-24T20:46:33,615 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a229fcffe6043a789c57a6e1df07697, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732481191977 2024-11-24T20:46:33,615 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting e35c760e29394812b5c68d17da6b4dad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732481192639 2024-11-24T20:46:33,615 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting b71cc8f2c1ae4ba3b055094863b94bae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732481192639 2024-11-24T20:46:33,616 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 687ef162984d4a10bbc54d6acd5bdc19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732481193021 2024-11-24T20:46:33,616 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting d5ba3f1169364ee9a247aa21796c7abf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732481193021 2024-11-24T20:46:33,639 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ac72b5882ff154796d9a89886bf5150#A#compaction#30 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:46:33,640 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/547287a95bf14a1da393ced68803e51b is 50, key is test_row_0/A:col10/1732481193031/Put/seqid=0 2024-11-24T20:46:33,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T20:46:33,646 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:46:33,647 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:46:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-24T20:46:33,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T20:46:33,653 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:46:33,656 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ac72b5882ff154796d9a89886bf5150#B#compaction#31 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:46:33,657 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/fa29ce8ab5b34baabb6c122878c274ae is 50, key is test_row_0/B:col10/1732481193031/Put/seqid=0 2024-11-24T20:46:33,659 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:46:33,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:46:33,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741867_1043 (size=12493) 2024-11-24T20:46:33,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741868_1044 (size=12493) 2024-11-24T20:46:33,700 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/fa29ce8ab5b34baabb6c122878c274ae as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/fa29ce8ab5b34baabb6c122878c274ae 2024-11-24T20:46:33,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:33,713 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:46:33,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ac72b5882ff154796d9a89886bf5150 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T20:46:33,721 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ac72b5882ff154796d9a89886bf5150/B of 6ac72b5882ff154796d9a89886bf5150 into fa29ce8ab5b34baabb6c122878c274ae(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:46:33,721 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:33,722 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., storeName=6ac72b5882ff154796d9a89886bf5150/B, priority=12, startTime=1732481193608; duration=0sec 2024-11-24T20:46:33,722 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:46:33,723 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:B 2024-11-24T20:46:33,723 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:46:33,727 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:46:33,727 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 6ac72b5882ff154796d9a89886bf5150/C is initiating minor compaction (all files) 2024-11-24T20:46:33,727 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ac72b5882ff154796d9a89886bf5150/C in TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:33,727 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/030d3bc82f5344aa86b3189f5a62fc7d, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/98310e8844ce4638bc14629709b63152, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/9e613f26f0fa48b7be95635a01ffbf68] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp, totalSize=35.5 K 2024-11-24T20:46:33,728 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 030d3bc82f5344aa86b3189f5a62fc7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732481191977 2024-11-24T20:46:33,730 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 98310e8844ce4638bc14629709b63152, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732481192639 2024-11-24T20:46:33,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/68497b1299d14f6481837507215c8af9 is 50, key is test_row_0/A:col10/1732481193373/Put/seqid=0 2024-11-24T20:46:33,730 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e613f26f0fa48b7be95635a01ffbf68, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732481193021 2024-11-24T20:46:33,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741869_1045 (size=14541) 2024-11-24T20:46:33,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/68497b1299d14f6481837507215c8af9 2024-11-24T20:46:33,755 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ac72b5882ff154796d9a89886bf5150#C#compaction#33 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:46:33,755 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/8abd57ebbb0c4a6ca919092ed5312322 is 50, key is test_row_0/C:col10/1732481193031/Put/seqid=0 2024-11-24T20:46:33,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T20:46:33,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/16b2b0207b78412b8b696d2795bf7083 is 50, key is test_row_0/B:col10/1732481193373/Put/seqid=0 2024-11-24T20:46:33,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741870_1046 (size=12493) 2024-11-24T20:46:33,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741871_1047 (size=12151) 2024-11-24T20:46:33,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:33,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481253796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:33,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:33,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:33,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481253800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:33,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:33,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481253804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:33,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:33,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:33,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:33,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481253806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:33,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:33,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:33,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481253809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-24T20:46:33,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:33,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:33,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:33,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:33,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:33,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:33,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:33,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:33,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481253915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:33,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:33,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:33,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481253915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:33,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:33,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:33,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:33,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481253916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:33,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481253915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:33,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:33,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:33,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:33,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481253925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:33,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:33,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:33,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T20:46:33,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-24T20:46:33,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:33,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:33,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:33,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:33,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:33,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:34,101 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/547287a95bf14a1da393ced68803e51b as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/547287a95bf14a1da393ced68803e51b 2024-11-24T20:46:34,121 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ac72b5882ff154796d9a89886bf5150/A of 6ac72b5882ff154796d9a89886bf5150 into 547287a95bf14a1da393ced68803e51b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:46:34,121 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:34,121 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., storeName=6ac72b5882ff154796d9a89886bf5150/A, priority=12, startTime=1732481193608; duration=0sec 2024-11-24T20:46:34,121 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:46:34,121 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:A 2024-11-24T20:46:34,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-24T20:46:34,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:34,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:34,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:34,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:34,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:34,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:34,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481254126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:34,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:34,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:34,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481254126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:34,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:34,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:34,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481254129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:34,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:34,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:34,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481254129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:34,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:34,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:34,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481254135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:34,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:34,201 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/8abd57ebbb0c4a6ca919092ed5312322 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/8abd57ebbb0c4a6ca919092ed5312322 2024-11-24T20:46:34,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/16b2b0207b78412b8b696d2795bf7083 2024-11-24T20:46:34,233 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ac72b5882ff154796d9a89886bf5150/C of 6ac72b5882ff154796d9a89886bf5150 into 8abd57ebbb0c4a6ca919092ed5312322(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:46:34,233 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:34,233 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., storeName=6ac72b5882ff154796d9a89886bf5150/C, priority=13, startTime=1732481193609; duration=0sec 2024-11-24T20:46:34,233 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:46:34,233 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:C 2024-11-24T20:46:34,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/5c997faf8a24435ca6dabfaaa322378e is 50, key is test_row_0/C:col10/1732481193373/Put/seqid=0 2024-11-24T20:46:34,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T20:46:34,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-24T20:46:34,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:34,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741872_1048 (size=12151) 2024-11-24T20:46:34,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:34,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:34,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:34,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:34,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:34,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/5c997faf8a24435ca6dabfaaa322378e 2024-11-24T20:46:34,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/68497b1299d14f6481837507215c8af9 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/68497b1299d14f6481837507215c8af9 2024-11-24T20:46:34,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/68497b1299d14f6481837507215c8af9, entries=200, sequenceid=168, filesize=14.2 K 2024-11-24T20:46:34,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/16b2b0207b78412b8b696d2795bf7083 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/16b2b0207b78412b8b696d2795bf7083 2024-11-24T20:46:34,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/16b2b0207b78412b8b696d2795bf7083, entries=150, sequenceid=168, filesize=11.9 K 2024-11-24T20:46:34,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/5c997faf8a24435ca6dabfaaa322378e as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/5c997faf8a24435ca6dabfaaa322378e 2024-11-24T20:46:34,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/5c997faf8a24435ca6dabfaaa322378e, entries=150, sequenceid=168, filesize=11.9 K 2024-11-24T20:46:34,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 6ac72b5882ff154796d9a89886bf5150 in 654ms, sequenceid=168, compaction requested=false 2024-11-24T20:46:34,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:34,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-24T20:46:34,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:34,445 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 6ac72b5882ff154796d9a89886bf5150 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-24T20:46:34,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:34,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:34,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:34,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:34,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481254455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:34,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481254455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:34,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:34,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:34,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481254456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:34,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:34,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:34,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:34,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481254462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:34,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/38f7af416d334719a07b55a49aa26f1a is 50, key is test_row_0/A:col10/1732481193804/Put/seqid=0 2024-11-24T20:46:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:34,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481254463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:34,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:34,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741873_1049 (size=12151) 2024-11-24T20:46:34,480 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/38f7af416d334719a07b55a49aa26f1a 2024-11-24T20:46:34,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/a131b0f05d0e41389ebea646ee192d86 is 50, key is test_row_0/B:col10/1732481193804/Put/seqid=0 2024-11-24T20:46:34,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741874_1050 (size=12151) 2024-11-24T20:46:34,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:34,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481254565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:34,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:34,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:34,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481254576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:34,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:34,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:34,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481254576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:34,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:34,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481254576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:34,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:34,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481254775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:34,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T20:46:34,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:34,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481254785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481254786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:34,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:34,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:34,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481254786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:34,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:34,930 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/a131b0f05d0e41389ebea646ee192d86 2024-11-24T20:46:34,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/e2bec717732647cba205012a7653c6f2 is 50, key is test_row_0/C:col10/1732481193804/Put/seqid=0 2024-11-24T20:46:34,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741875_1051 (size=12151) 2024-11-24T20:46:34,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481254975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:34,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:34,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:35,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:35,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481255085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:35,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:35,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:35,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:35,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:35,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:35,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481255095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:35,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481255095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:35,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:35,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481255096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:35,352 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/e2bec717732647cba205012a7653c6f2 2024-11-24T20:46:35,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/38f7af416d334719a07b55a49aa26f1a as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/38f7af416d334719a07b55a49aa26f1a 2024-11-24T20:46:35,374 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/38f7af416d334719a07b55a49aa26f1a, entries=150, sequenceid=196, filesize=11.9 K 2024-11-24T20:46:35,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/a131b0f05d0e41389ebea646ee192d86 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/a131b0f05d0e41389ebea646ee192d86 2024-11-24T20:46:35,388 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/a131b0f05d0e41389ebea646ee192d86, entries=150, sequenceid=196, filesize=11.9 K 2024-11-24T20:46:35,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/e2bec717732647cba205012a7653c6f2 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/e2bec717732647cba205012a7653c6f2 2024-11-24T20:46:35,399 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/e2bec717732647cba205012a7653c6f2, entries=150, sequenceid=196, filesize=11.9 K 2024-11-24T20:46:35,401 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 6ac72b5882ff154796d9a89886bf5150 in 956ms, sequenceid=196, compaction requested=true 2024-11-24T20:46:35,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:35,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:35,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-24T20:46:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-24T20:46:35,408 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-24T20:46:35,408 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7450 sec 2024-11-24T20:46:35,414 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.7630 sec 2024-11-24T20:46:35,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:35,597 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:46:35,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ac72b5882ff154796d9a89886bf5150 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-24T20:46:35,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/e1fe23248f42416886771f25e20739e2 is 50, key is test_row_0/A:col10/1732481194450/Put/seqid=0 2024-11-24T20:46:35,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741876_1052 (size=12151) 2024-11-24T20:46:35,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:35,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481255647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:35,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:35,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:35,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:35,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481255648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:35,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:35,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:35,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:35,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:35,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481255653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:35,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:35,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481255654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:35,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481255765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:35,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:35,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:35,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:35,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481255765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:35,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:35,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:35,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:35,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481255765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:35,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:35,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:35,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:35,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481255765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:35,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:35,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:35,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T20:46:35,795 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:46:35,796 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:46:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-24T20:46:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T20:46:35,798 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:46:35,808 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:46:35,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:46:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T20:46:35,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T20:46:35,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:35,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:35,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:35,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:35,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:35,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:35,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:35,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481255975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:35,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:35,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:35,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:35,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481255975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:35,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481255976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:35,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:35,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:35,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:35,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:35,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:35,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:35,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481255976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:35,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:35,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:35,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:35,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:35,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481255995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:36,002 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_0' in 6ac72b5882ff154796d9a89886bf5150 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:46:36,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:36,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/e1fe23248f42416886771f25e20739e2 2024-11-24T20:46:36,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/8439df660eec455ab3dd2e62fdf7bc17 is 50, key is test_row_0/B:col10/1732481194450/Put/seqid=0 2024-11-24T20:46:36,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741877_1053 (size=12151) 2024-11-24T20:46:36,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/8439df660eec455ab3dd2e62fdf7bc17 2024-11-24T20:46:36,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/f36fd6a46b4044a9a98581832c33e332 is 50, key is test_row_0/C:col10/1732481194450/Put/seqid=0 2024-11-24T20:46:36,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741878_1054 (size=12151) 2024-11-24T20:46:36,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/f36fd6a46b4044a9a98581832c33e332 2024-11-24T20:46:36,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/e1fe23248f42416886771f25e20739e2 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/e1fe23248f42416886771f25e20739e2 2024-11-24T20:46:36,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/e1fe23248f42416886771f25e20739e2, entries=150, sequenceid=208, filesize=11.9 K 2024-11-24T20:46:36,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/8439df660eec455ab3dd2e62fdf7bc17 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/8439df660eec455ab3dd2e62fdf7bc17 2024-11-24T20:46:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T20:46:36,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T20:46:36,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:36,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:36,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:36,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:36,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:36,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:36,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/8439df660eec455ab3dd2e62fdf7bc17, entries=150, sequenceid=208, filesize=11.9 K 2024-11-24T20:46:36,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/f36fd6a46b4044a9a98581832c33e332 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/f36fd6a46b4044a9a98581832c33e332 2024-11-24T20:46:36,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/f36fd6a46b4044a9a98581832c33e332, entries=150, sequenceid=208, filesize=11.9 K 2024-11-24T20:46:36,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 6ac72b5882ff154796d9a89886bf5150 in 539ms, sequenceid=208, compaction requested=true 2024-11-24T20:46:36,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:36,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:46:36,137 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:46:36,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:46:36,137 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:46:36,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:46:36,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:46:36,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:46:36,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:46:36,139 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:46:36,140 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 6ac72b5882ff154796d9a89886bf5150/B is initiating minor compaction (all files) 2024-11-24T20:46:36,140 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ac72b5882ff154796d9a89886bf5150/B in TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:36,140 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/fa29ce8ab5b34baabb6c122878c274ae, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/16b2b0207b78412b8b696d2795bf7083, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/a131b0f05d0e41389ebea646ee192d86, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/8439df660eec455ab3dd2e62fdf7bc17] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp, totalSize=47.8 K 2024-11-24T20:46:36,140 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51336 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:46:36,140 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 6ac72b5882ff154796d9a89886bf5150/A is initiating minor compaction (all files) 2024-11-24T20:46:36,140 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ac72b5882ff154796d9a89886bf5150/A in TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:36,140 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/547287a95bf14a1da393ced68803e51b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/68497b1299d14f6481837507215c8af9, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/38f7af416d334719a07b55a49aa26f1a, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/e1fe23248f42416886771f25e20739e2] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp, totalSize=50.1 K 2024-11-24T20:46:36,141 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting fa29ce8ab5b34baabb6c122878c274ae, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732481193021 2024-11-24T20:46:36,141 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 547287a95bf14a1da393ced68803e51b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732481193021 2024-11-24T20:46:36,143 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68497b1299d14f6481837507215c8af9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732481193373 2024-11-24T20:46:36,143 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 16b2b0207b78412b8b696d2795bf7083, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732481193373 2024-11-24T20:46:36,143 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38f7af416d334719a07b55a49aa26f1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732481193802 2024-11-24T20:46:36,144 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting a131b0f05d0e41389ebea646ee192d86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732481193802 2024-11-24T20:46:36,144 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1fe23248f42416886771f25e20739e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732481194450 2024-11-24T20:46:36,144 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 8439df660eec455ab3dd2e62fdf7bc17, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732481194450 2024-11-24T20:46:36,165 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ac72b5882ff154796d9a89886bf5150#A#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:46:36,166 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/e7a7175f5fba4bcf8912f2fa2674db19 is 50, key is test_row_0/A:col10/1732481194450/Put/seqid=0 2024-11-24T20:46:36,175 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ac72b5882ff154796d9a89886bf5150#B#compaction#43 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:46:36,175 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/e43225079a3649699344a78fd85949d2 is 50, key is test_row_0/B:col10/1732481194450/Put/seqid=0 2024-11-24T20:46:36,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741879_1055 (size=12629) 2024-11-24T20:46:36,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741880_1056 (size=12629) 2024-11-24T20:46:36,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T20:46:36,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:36,273 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 6ac72b5882ff154796d9a89886bf5150 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-24T20:46:36,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/b34a936810424b659068699c3554dcb1 is 50, key is test_row_0/A:col10/1732481195645/Put/seqid=0 2024-11-24T20:46:36,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:36,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:36,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741881_1057 (size=12151) 2024-11-24T20:46:36,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:36,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481256299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:36,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:36,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:36,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481256301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:36,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:36,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:36,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481256301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:36,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:36,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481256303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:36,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:36,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:36,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:36,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481256405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:36,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:36,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:36,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481256415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:36,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:36,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481256415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:36,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:36,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T20:46:36,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:36,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481256425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:36,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:36,602 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/e7a7175f5fba4bcf8912f2fa2674db19 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/e7a7175f5fba4bcf8912f2fa2674db19 2024-11-24T20:46:36,606 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/e43225079a3649699344a78fd85949d2 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/e43225079a3649699344a78fd85949d2 2024-11-24T20:46:36,610 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ac72b5882ff154796d9a89886bf5150/A of 6ac72b5882ff154796d9a89886bf5150 into e7a7175f5fba4bcf8912f2fa2674db19(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:46:36,610 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:36,610 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., storeName=6ac72b5882ff154796d9a89886bf5150/A, priority=12, startTime=1732481196137; duration=0sec 2024-11-24T20:46:36,611 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:46:36,611 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:A 2024-11-24T20:46:36,611 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:46:36,613 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:46:36,613 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 6ac72b5882ff154796d9a89886bf5150/C is initiating minor compaction (all files) 2024-11-24T20:46:36,613 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ac72b5882ff154796d9a89886bf5150/C in TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:36,613 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/8abd57ebbb0c4a6ca919092ed5312322, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/5c997faf8a24435ca6dabfaaa322378e, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/e2bec717732647cba205012a7653c6f2, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/f36fd6a46b4044a9a98581832c33e332] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp, totalSize=47.8 K 2024-11-24T20:46:36,614 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8abd57ebbb0c4a6ca919092ed5312322, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732481193021 2024-11-24T20:46:36,614 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c997faf8a24435ca6dabfaaa322378e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732481193373 2024-11-24T20:46:36,616 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2bec717732647cba205012a7653c6f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732481193802 2024-11-24T20:46:36,617 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ac72b5882ff154796d9a89886bf5150/B of 6ac72b5882ff154796d9a89886bf5150 into e43225079a3649699344a78fd85949d2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:46:36,618 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:36,618 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting f36fd6a46b4044a9a98581832c33e332, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732481194450 2024-11-24T20:46:36,618 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., storeName=6ac72b5882ff154796d9a89886bf5150/B, priority=12, startTime=1732481196137; duration=0sec 2024-11-24T20:46:36,618 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:46:36,618 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:B 2024-11-24T20:46:36,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481256615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:36,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:36,629 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ac72b5882ff154796d9a89886bf5150#C#compaction#45 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:46:36,630 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/2ceeafc6d89542e58275bf65c885c641 is 50, key is test_row_0/C:col10/1732481194450/Put/seqid=0 2024-11-24T20:46:36,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481256625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481256626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:36,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:36,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:36,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481256635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:36,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:36,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741882_1058 (size=12629) 2024-11-24T20:46:36,647 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/2ceeafc6d89542e58275bf65c885c641 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/2ceeafc6d89542e58275bf65c885c641 2024-11-24T20:46:36,655 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ac72b5882ff154796d9a89886bf5150/C of 6ac72b5882ff154796d9a89886bf5150 into 2ceeafc6d89542e58275bf65c885c641(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:46:36,655 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:36,655 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., storeName=6ac72b5882ff154796d9a89886bf5150/C, priority=12, startTime=1732481196138; duration=0sec 2024-11-24T20:46:36,655 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:46:36,655 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:C 2024-11-24T20:46:36,695 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/b34a936810424b659068699c3554dcb1 2024-11-24T20:46:36,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/3f75ecfa84cf475f84282b33c13380f4 is 50, key is test_row_0/B:col10/1732481195645/Put/seqid=0 2024-11-24T20:46:36,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741883_1059 (size=12151) 2024-11-24T20:46:36,713 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/3f75ecfa84cf475f84282b33c13380f4 2024-11-24T20:46:36,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/5ae3f7f48f7d4552a13b61387ef7ed02 is 50, key is test_row_0/C:col10/1732481195645/Put/seqid=0 2024-11-24T20:46:36,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741884_1060 (size=12151) 2024-11-24T20:46:36,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T20:46:36,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:36,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481256935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:36,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:36,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:36,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481256945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:36,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:36,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:36,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481256945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:36,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:36,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481256947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:36,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:36,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:36,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:37,143 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/5ae3f7f48f7d4552a13b61387ef7ed02 2024-11-24T20:46:37,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/b34a936810424b659068699c3554dcb1 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/b34a936810424b659068699c3554dcb1 2024-11-24T20:46:37,155 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/b34a936810424b659068699c3554dcb1, entries=150, sequenceid=233, filesize=11.9 K 2024-11-24T20:46:37,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/3f75ecfa84cf475f84282b33c13380f4 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/3f75ecfa84cf475f84282b33c13380f4 2024-11-24T20:46:37,163 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/3f75ecfa84cf475f84282b33c13380f4, entries=150, sequenceid=233, filesize=11.9 K 2024-11-24T20:46:37,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/5ae3f7f48f7d4552a13b61387ef7ed02 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/5ae3f7f48f7d4552a13b61387ef7ed02 2024-11-24T20:46:37,173 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/5ae3f7f48f7d4552a13b61387ef7ed02, entries=150, sequenceid=233, filesize=11.9 K 2024-11-24T20:46:37,174 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 6ac72b5882ff154796d9a89886bf5150 in 902ms, sequenceid=233, compaction requested=false 2024-11-24T20:46:37,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:37,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:37,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-24T20:46:37,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-24T20:46:37,180 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-24T20:46:37,180 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3680 sec 2024-11-24T20:46:37,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.3860 sec 2024-11-24T20:46:37,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:37,448 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:46:37,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ac72b5882ff154796d9a89886bf5150 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-24T20:46:37,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/19d7bfb781c543b5a06705c8e1fe3f6d is 50, key is test_row_0/A:col10/1732481196301/Put/seqid=0 2024-11-24T20:46:37,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741885_1061 (size=12151) 2024-11-24T20:46:37,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/19d7bfb781c543b5a06705c8e1fe3f6d 2024-11-24T20:46:37,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/f3f7d2a5742b483888a72aa6636f178d is 50, key is test_row_0/B:col10/1732481196301/Put/seqid=0 2024-11-24T20:46:37,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741886_1062 (size=12151) 2024-11-24T20:46:37,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/f3f7d2a5742b483888a72aa6636f178d 2024-11-24T20:46:37,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:37,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481257504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:37,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:37,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:37,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:37,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481257505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:37,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:37,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:37,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:37,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481257507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:37,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:37,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:37,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:37,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:37,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481257509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:37,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:37,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:37,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:37,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/6b14590d0747467ca90ad7eb50e34704 is 50, key is test_row_0/C:col10/1732481196301/Put/seqid=0 2024-11-24T20:46:37,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741887_1063 (size=12151) 2024-11-24T20:46:37,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:37,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481257615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:37,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:37,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:37,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:37,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:37,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481257615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:37,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:37,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:37,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:37,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:37,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481257615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:37,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:37,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:37,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:37,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:37,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481257625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:37,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:37,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:37,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:37,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:37,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481257825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:37,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:37,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:37,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:37,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:37,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481257825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:37,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481257826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:37,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:37,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:37,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:37,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:37,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:37,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:37,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:37,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481257835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:37,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:37,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:37,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:37,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/6b14590d0747467ca90ad7eb50e34704 2024-11-24T20:46:37,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/19d7bfb781c543b5a06705c8e1fe3f6d as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/19d7bfb781c543b5a06705c8e1fe3f6d 2024-11-24T20:46:37,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T20:46:37,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/19d7bfb781c543b5a06705c8e1fe3f6d, entries=150, sequenceid=248, filesize=11.9 K 2024-11-24T20:46:37,946 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:46:37,947 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:46:37,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/f3f7d2a5742b483888a72aa6636f178d as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/f3f7d2a5742b483888a72aa6636f178d 2024-11-24T20:46:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-24T20:46:37,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T20:46:37,950 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:46:37,951 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:46:37,951 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:46:37,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/f3f7d2a5742b483888a72aa6636f178d, entries=150, sequenceid=248, filesize=11.9 K 2024-11-24T20:46:37,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/6b14590d0747467ca90ad7eb50e34704 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/6b14590d0747467ca90ad7eb50e34704 2024-11-24T20:46:37,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/6b14590d0747467ca90ad7eb50e34704, entries=150, sequenceid=248, filesize=11.9 K 2024-11-24T20:46:37,965 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 6ac72b5882ff154796d9a89886bf5150 in 517ms, sequenceid=248, compaction requested=true 2024-11-24T20:46:37,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:37,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:46:37,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:46:37,966 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:46:37,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:46:37,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:46:37,966 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:46:37,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:46:37,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:46:37,967 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:46:37,967 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 6ac72b5882ff154796d9a89886bf5150/A is initiating minor compaction (all files) 2024-11-24T20:46:37,967 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ac72b5882ff154796d9a89886bf5150/A in TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:37,967 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/e7a7175f5fba4bcf8912f2fa2674db19, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/b34a936810424b659068699c3554dcb1, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/19d7bfb781c543b5a06705c8e1fe3f6d] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp, totalSize=36.1 K 2024-11-24T20:46:37,967 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:46:37,967 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 6ac72b5882ff154796d9a89886bf5150/B is initiating minor compaction (all files) 2024-11-24T20:46:37,968 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ac72b5882ff154796d9a89886bf5150/B in TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:37,968 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/e43225079a3649699344a78fd85949d2, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/3f75ecfa84cf475f84282b33c13380f4, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/f3f7d2a5742b483888a72aa6636f178d] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp, totalSize=36.1 K 2024-11-24T20:46:37,968 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting e43225079a3649699344a78fd85949d2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732481194450 2024-11-24T20:46:37,968 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7a7175f5fba4bcf8912f2fa2674db19, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732481194450 2024-11-24T20:46:37,969 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f75ecfa84cf475f84282b33c13380f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732481195645 2024-11-24T20:46:37,969 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting b34a936810424b659068699c3554dcb1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732481195645 2024-11-24T20:46:37,970 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting f3f7d2a5742b483888a72aa6636f178d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732481196299 2024-11-24T20:46:37,970 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19d7bfb781c543b5a06705c8e1fe3f6d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732481196299 2024-11-24T20:46:37,979 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ac72b5882ff154796d9a89886bf5150#A#compaction#51 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:46:37,980 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ac72b5882ff154796d9a89886bf5150#B#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:46:37,980 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/f418a088736149d09d1cdd90c069f28b is 50, key is test_row_0/A:col10/1732481196301/Put/seqid=0 2024-11-24T20:46:37,981 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/3db2440010724faa9af33117f0f15f98 is 50, key is test_row_0/B:col10/1732481196301/Put/seqid=0 2024-11-24T20:46:37,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741888_1064 (size=12731) 2024-11-24T20:46:38,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741889_1065 (size=12731) 2024-11-24T20:46:38,010 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/3db2440010724faa9af33117f0f15f98 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/3db2440010724faa9af33117f0f15f98 2024-11-24T20:46:38,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:38,019 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:46:38,019 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ac72b5882ff154796d9a89886bf5150 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-24T20:46:38,019 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ac72b5882ff154796d9a89886bf5150/B of 6ac72b5882ff154796d9a89886bf5150 into 3db2440010724faa9af33117f0f15f98(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:46:38,019 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:38,019 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., storeName=6ac72b5882ff154796d9a89886bf5150/B, priority=13, startTime=1732481197966; duration=0sec 2024-11-24T20:46:38,020 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:46:38,020 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:B 2024-11-24T20:46:38,020 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:46:38,023 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:46:38,023 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 6ac72b5882ff154796d9a89886bf5150/C is initiating minor compaction (all files) 2024-11-24T20:46:38,023 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ac72b5882ff154796d9a89886bf5150/C in TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:38,023 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/2ceeafc6d89542e58275bf65c885c641, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/5ae3f7f48f7d4552a13b61387ef7ed02, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/6b14590d0747467ca90ad7eb50e34704] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp, totalSize=36.1 K 2024-11-24T20:46:38,024 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ceeafc6d89542e58275bf65c885c641, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732481194450 2024-11-24T20:46:38,025 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ae3f7f48f7d4552a13b61387ef7ed02, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732481195645 2024-11-24T20:46:38,025 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b14590d0747467ca90ad7eb50e34704, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732481196299 2024-11-24T20:46:38,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/2e846dcb03bc449fab36c78408d8c10f is 50, key is test_row_0/A:col10/1732481198018/Put/seqid=0 2024-11-24T20:46:38,054 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ac72b5882ff154796d9a89886bf5150#C#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:46:38,054 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/e79286b28bbc4d909d27475b51379e65 is 50, key is test_row_0/C:col10/1732481196301/Put/seqid=0 2024-11-24T20:46:38,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T20:46:38,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741890_1066 (size=14741) 2024-11-24T20:46:38,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/2e846dcb03bc449fab36c78408d8c10f 2024-11-24T20:46:38,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/1190e90ebaa3453db4d913453a0f3a79 is 50, key is test_row_0/B:col10/1732481198018/Put/seqid=0 2024-11-24T20:46:38,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741891_1067 (size=12731) 2024-11-24T20:46:38,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:38,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481258075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:38,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:38,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741892_1068 (size=12301) 2024-11-24T20:46:38,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/1190e90ebaa3453db4d913453a0f3a79 2024-11-24T20:46:38,093 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/e79286b28bbc4d909d27475b51379e65 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/e79286b28bbc4d909d27475b51379e65 2024-11-24T20:46:38,105 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ac72b5882ff154796d9a89886bf5150/C of 6ac72b5882ff154796d9a89886bf5150 into e79286b28bbc4d909d27475b51379e65(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:46:38,105 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:38,105 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., storeName=6ac72b5882ff154796d9a89886bf5150/C, priority=13, startTime=1732481197966; duration=0sec 2024-11-24T20:46:38,105 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:46:38,105 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:C 2024-11-24T20:46:38,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T20:46:38,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/43027813598b47c884b350d24cf7d9b0 is 50, key is test_row_0/C:col10/1732481198018/Put/seqid=0 2024-11-24T20:46:38,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:38,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:38,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:38,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:38,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:38,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:38,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741893_1069 (size=12301) 2024-11-24T20:46:38,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:38,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:38,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481258135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:38,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481258135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:38,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481258136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:38,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:38,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481258145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:38,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:38,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:38,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481258185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:38,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:38,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T20:46:38,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T20:46:38,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:38,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:38,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:38,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:38,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:38,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:38,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:38,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481258395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:38,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:38,400 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/f418a088736149d09d1cdd90c069f28b as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/f418a088736149d09d1cdd90c069f28b 2024-11-24T20:46:38,408 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ac72b5882ff154796d9a89886bf5150/A of 6ac72b5882ff154796d9a89886bf5150 into f418a088736149d09d1cdd90c069f28b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:46:38,408 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:38,408 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., storeName=6ac72b5882ff154796d9a89886bf5150/A, priority=13, startTime=1732481197965; duration=0sec 2024-11-24T20:46:38,408 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:46:38,408 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:A 2024-11-24T20:46:38,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T20:46:38,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:38,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:38,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:38,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:38,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:38,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:38,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/43027813598b47c884b350d24cf7d9b0 2024-11-24T20:46:38,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/2e846dcb03bc449fab36c78408d8c10f as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/2e846dcb03bc449fab36c78408d8c10f 2024-11-24T20:46:38,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/2e846dcb03bc449fab36c78408d8c10f, entries=200, sequenceid=275, filesize=14.4 K 2024-11-24T20:46:38,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/1190e90ebaa3453db4d913453a0f3a79 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/1190e90ebaa3453db4d913453a0f3a79 2024-11-24T20:46:38,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/1190e90ebaa3453db4d913453a0f3a79, entries=150, sequenceid=275, filesize=12.0 K 2024-11-24T20:46:38,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/43027813598b47c884b350d24cf7d9b0 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/43027813598b47c884b350d24cf7d9b0 2024-11-24T20:46:38,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/43027813598b47c884b350d24cf7d9b0, entries=150, sequenceid=275, filesize=12.0 K 2024-11-24T20:46:38,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 6ac72b5882ff154796d9a89886bf5150 in 535ms, sequenceid=275, compaction requested=false 2024-11-24T20:46:38,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:38,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T20:46:38,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T20:46:38,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:38,584 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 6ac72b5882ff154796d9a89886bf5150 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-24T20:46:38,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/78a45aebe0ab45729b982da9fa0fc7f7 is 50, key is test_row_0/A:col10/1732481198072/Put/seqid=0 2024-11-24T20:46:38,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741894_1070 (size=12301) 2024-11-24T20:46:38,612 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/78a45aebe0ab45729b982da9fa0fc7f7 2024-11-24T20:46:38,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/d83c00a7823c48db86ef1d3cf9c254d3 is 50, key is test_row_0/B:col10/1732481198072/Put/seqid=0 2024-11-24T20:46:38,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741895_1071 (size=12301) 2024-11-24T20:46:38,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:38,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:38,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481258677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:38,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:38,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481258680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:38,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:38,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481258682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:38,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:38,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481258683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:38,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:38,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:38,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:38,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:38,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481258705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:38,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:38,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:38,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481258785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:38,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:38,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:38,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481258795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:38,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:38,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481258795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:38,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:38,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:38,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481258796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:38,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:38,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:38,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481258995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:39,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481259005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:39,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481259005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:39,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:39,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:39,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481259005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:39,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:39,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:39,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:39,032 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/d83c00a7823c48db86ef1d3cf9c254d3 2024-11-24T20:46:39,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/968eaba0d1b94278a84ed5e72d36e5b2 is 50, key is test_row_0/C:col10/1732481198072/Put/seqid=0 2024-11-24T20:46:39,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741896_1072 (size=12301) 2024-11-24T20:46:39,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T20:46:39,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:39,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481259215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:39,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:39,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:39,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:39,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481259305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:39,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:39,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:39,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:39,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481259315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:39,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481259315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:39,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:39,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:39,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:39,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481259315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:39,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:39,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:39,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:39,445 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/968eaba0d1b94278a84ed5e72d36e5b2 2024-11-24T20:46:39,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/78a45aebe0ab45729b982da9fa0fc7f7 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/78a45aebe0ab45729b982da9fa0fc7f7 2024-11-24T20:46:39,459 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/78a45aebe0ab45729b982da9fa0fc7f7, entries=150, sequenceid=287, filesize=12.0 K 2024-11-24T20:46:39,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/d83c00a7823c48db86ef1d3cf9c254d3 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/d83c00a7823c48db86ef1d3cf9c254d3 2024-11-24T20:46:39,466 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/d83c00a7823c48db86ef1d3cf9c254d3, entries=150, sequenceid=287, filesize=12.0 K 2024-11-24T20:46:39,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/968eaba0d1b94278a84ed5e72d36e5b2 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/968eaba0d1b94278a84ed5e72d36e5b2 2024-11-24T20:46:39,473 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/968eaba0d1b94278a84ed5e72d36e5b2, entries=150, sequenceid=287, filesize=12.0 K 2024-11-24T20:46:39,474 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 6ac72b5882ff154796d9a89886bf5150 in 890ms, sequenceid=287, compaction requested=true 2024-11-24T20:46:39,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:39,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:39,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-24T20:46:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-24T20:46:39,477 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-24T20:46:39,477 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5250 sec 2024-11-24T20:46:39,479 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.5310 sec 2024-11-24T20:46:39,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:39,828 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:46:39,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ac72b5882ff154796d9a89886bf5150 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-24T20:46:39,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/67cb65dd2bf34dd48dcbf4fff6133650 is 50, key is test_row_0/A:col10/1732481198681/Put/seqid=0 2024-11-24T20:46:39,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:39,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481259833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481259834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:39,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481259836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:39,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:39,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:39,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:39,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:39,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:39,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:39,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:39,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:39,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:39,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481259838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:39,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:39,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:39,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:39,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741897_1073 (size=14741) 2024-11-24T20:46:39,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:39,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481259945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:39,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:39,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:39,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:39,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:39,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481259945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:39,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:39,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:39,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:39,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481259945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:39,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:39,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:39,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:39,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481259955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:39,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:39,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:39,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T20:46:40,095 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:46:40,096 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:46:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-24T20:46:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T20:46:40,098 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:46:40,099 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:46:40,099 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:46:40,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481260155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:40,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481260155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:40,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:40,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:40,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481260156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:40,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:40,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:40,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:40,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:40,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481260165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:40,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:40,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:40,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T20:46:40,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:40,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481260235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:40,237 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_0' in 6ac72b5882ff154796d9a89886bf5150 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:46:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:40,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T20:46:40,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:40,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:40,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:40,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:40,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:40,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:40,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/67cb65dd2bf34dd48dcbf4fff6133650 2024-11-24T20:46:40,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/2177a3e521c741ac9ef5382dbb69cc67 is 50, key is test_row_0/B:col10/1732481198681/Put/seqid=0 2024-11-24T20:46:40,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741898_1074 (size=12301) 2024-11-24T20:46:40,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T20:46:40,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:40,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:40,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:40,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:40,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:40,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:40,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T20:46:40,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:40,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481260465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:40,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:40,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:40,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:40,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:40,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481260466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:40,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:40,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:40,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:40,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:40,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481260466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:40,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:40,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481260475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:40,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:40,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:40,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T20:46:40,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:40,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:40,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:40,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:40,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:40,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:40,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/2177a3e521c741ac9ef5382dbb69cc67 2024-11-24T20:46:40,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/e48352d346024bceb7088a71d7af3ac8 is 50, key is test_row_0/C:col10/1732481198681/Put/seqid=0 2024-11-24T20:46:40,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T20:46:40,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:40,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:40,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:40,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:40,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:40,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T20:46:40,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741899_1075 (size=12301) 2024-11-24T20:46:40,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/e48352d346024bceb7088a71d7af3ac8 2024-11-24T20:46:40,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/67cb65dd2bf34dd48dcbf4fff6133650 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/67cb65dd2bf34dd48dcbf4fff6133650 2024-11-24T20:46:40,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/67cb65dd2bf34dd48dcbf4fff6133650, entries=200, sequenceid=315, filesize=14.4 K 2024-11-24T20:46:40,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/2177a3e521c741ac9ef5382dbb69cc67 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/2177a3e521c741ac9ef5382dbb69cc67 2024-11-24T20:46:40,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/2177a3e521c741ac9ef5382dbb69cc67, entries=150, sequenceid=315, filesize=12.0 K 2024-11-24T20:46:40,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/e48352d346024bceb7088a71d7af3ac8 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/e48352d346024bceb7088a71d7af3ac8 2024-11-24T20:46:40,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/e48352d346024bceb7088a71d7af3ac8, entries=150, sequenceid=315, filesize=12.0 K 2024-11-24T20:46:40,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 6ac72b5882ff154796d9a89886bf5150 in 942ms, sequenceid=315, compaction requested=true 2024-11-24T20:46:40,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:40,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:46:40,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:46:40,771 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:46:40,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:46:40,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:46:40,771 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:46:40,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:46:40,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:46:40,772 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54514 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:46:40,772 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 6ac72b5882ff154796d9a89886bf5150/A is initiating minor compaction (all files) 2024-11-24T20:46:40,772 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:46:40,773 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ac72b5882ff154796d9a89886bf5150/A in TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:40,773 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 6ac72b5882ff154796d9a89886bf5150/B is initiating minor compaction (all files) 2024-11-24T20:46:40,773 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ac72b5882ff154796d9a89886bf5150/B in TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:40,773 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/f418a088736149d09d1cdd90c069f28b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/2e846dcb03bc449fab36c78408d8c10f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/78a45aebe0ab45729b982da9fa0fc7f7, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/67cb65dd2bf34dd48dcbf4fff6133650] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp, totalSize=53.2 K 2024-11-24T20:46:40,773 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/3db2440010724faa9af33117f0f15f98, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/1190e90ebaa3453db4d913453a0f3a79, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/d83c00a7823c48db86ef1d3cf9c254d3, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/2177a3e521c741ac9ef5382dbb69cc67] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp, totalSize=48.5 K 2024-11-24T20:46:40,773 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 3db2440010724faa9af33117f0f15f98, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732481196299 2024-11-24T20:46:40,773 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting f418a088736149d09d1cdd90c069f28b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732481196299 2024-11-24T20:46:40,774 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 1190e90ebaa3453db4d913453a0f3a79, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732481197494 2024-11-24T20:46:40,774 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e846dcb03bc449fab36c78408d8c10f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732481197494 2024-11-24T20:46:40,775 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78a45aebe0ab45729b982da9fa0fc7f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732481198023 2024-11-24T20:46:40,775 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting d83c00a7823c48db86ef1d3cf9c254d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732481198023 2024-11-24T20:46:40,776 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67cb65dd2bf34dd48dcbf4fff6133650, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732481198678 2024-11-24T20:46:40,776 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 2177a3e521c741ac9ef5382dbb69cc67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732481198681 2024-11-24T20:46:40,799 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ac72b5882ff154796d9a89886bf5150#B#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:46:40,800 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/21e75c296d0241d49ed4f4dc92393b97 is 50, key is test_row_0/B:col10/1732481198681/Put/seqid=0 2024-11-24T20:46:40,804 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ac72b5882ff154796d9a89886bf5150#A#compaction#64 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:46:40,805 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/0dba1f2787ee419caa95e537c6db0f9c is 50, key is test_row_0/A:col10/1732481198681/Put/seqid=0 2024-11-24T20:46:40,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741900_1076 (size=13017) 2024-11-24T20:46:40,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741901_1077 (size=13017) 2024-11-24T20:46:40,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T20:46:40,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:40,875 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 6ac72b5882ff154796d9a89886bf5150 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-24T20:46:40,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/57850d210582427cb330dd82a9399ee9 is 50, key is test_row_0/A:col10/1732481199830/Put/seqid=0 2024-11-24T20:46:40,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741902_1078 (size=12301) 2024-11-24T20:46:40,901 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/57850d210582427cb330dd82a9399ee9 2024-11-24T20:46:40,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/89cc2077ed3041819d19304122acce5e is 50, key is test_row_0/B:col10/1732481199830/Put/seqid=0 2024-11-24T20:46:40,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741903_1079 (size=12301) 2024-11-24T20:46:40,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:40,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:41,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:41,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481261047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:41,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:41,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:41,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481261048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:41,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:41,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:41,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481261048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:41,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:41,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:41,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481261049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:41,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:41,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:41,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481261156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:41,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:41,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:41,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481261166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:41,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:41,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481261166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:41,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:41,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:41,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:41,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481261166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:41,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:41,224 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/21e75c296d0241d49ed4f4dc92393b97 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/21e75c296d0241d49ed4f4dc92393b97 2024-11-24T20:46:41,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T20:46:41,237 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/0dba1f2787ee419caa95e537c6db0f9c as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/0dba1f2787ee419caa95e537c6db0f9c 2024-11-24T20:46:41,240 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ac72b5882ff154796d9a89886bf5150/B of 6ac72b5882ff154796d9a89886bf5150 into 21e75c296d0241d49ed4f4dc92393b97(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:46:41,240 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:41,240 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., storeName=6ac72b5882ff154796d9a89886bf5150/B, priority=12, startTime=1732481200771; duration=0sec 2024-11-24T20:46:41,240 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:46:41,240 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:B 2024-11-24T20:46:41,240 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:46:41,245 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ac72b5882ff154796d9a89886bf5150/A of 6ac72b5882ff154796d9a89886bf5150 into 0dba1f2787ee419caa95e537c6db0f9c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:46:41,245 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:41,245 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., storeName=6ac72b5882ff154796d9a89886bf5150/A, priority=12, startTime=1732481200770; duration=0sec 2024-11-24T20:46:41,245 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:46:41,245 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:A 2024-11-24T20:46:41,246 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:46:41,246 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 6ac72b5882ff154796d9a89886bf5150/C is initiating minor compaction (all files) 2024-11-24T20:46:41,246 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ac72b5882ff154796d9a89886bf5150/C in TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:41,247 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/e79286b28bbc4d909d27475b51379e65, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/43027813598b47c884b350d24cf7d9b0, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/968eaba0d1b94278a84ed5e72d36e5b2, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/e48352d346024bceb7088a71d7af3ac8] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp, totalSize=48.5 K 2024-11-24T20:46:41,247 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting e79286b28bbc4d909d27475b51379e65, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732481196299 2024-11-24T20:46:41,247 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 43027813598b47c884b350d24cf7d9b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732481197494 2024-11-24T20:46:41,248 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 968eaba0d1b94278a84ed5e72d36e5b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732481198023 2024-11-24T20:46:41,248 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting e48352d346024bceb7088a71d7af3ac8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732481198681 2024-11-24T20:46:41,282 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ac72b5882ff154796d9a89886bf5150#C#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:46:41,283 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/8eb1ceff8d0f4f838df6b093aae97483 is 50, key is test_row_0/C:col10/1732481198681/Put/seqid=0 2024-11-24T20:46:41,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741904_1080 (size=13017) 2024-11-24T20:46:41,328 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/89cc2077ed3041819d19304122acce5e 2024-11-24T20:46:41,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/3fa73956fc594e39b682da4b3f0c5195 is 50, key is test_row_0/C:col10/1732481199830/Put/seqid=0 2024-11-24T20:46:41,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741905_1081 (size=12301) 2024-11-24T20:46:41,360 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/3fa73956fc594e39b682da4b3f0c5195 2024-11-24T20:46:41,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481261365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:41,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:41,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/57850d210582427cb330dd82a9399ee9 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/57850d210582427cb330dd82a9399ee9 2024-11-24T20:46:41,376 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/57850d210582427cb330dd82a9399ee9, entries=150, sequenceid=323, filesize=12.0 K 2024-11-24T20:46:41,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/89cc2077ed3041819d19304122acce5e as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/89cc2077ed3041819d19304122acce5e 2024-11-24T20:46:41,393 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/89cc2077ed3041819d19304122acce5e, entries=150, sequenceid=323, filesize=12.0 K 2024-11-24T20:46:41,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/3fa73956fc594e39b682da4b3f0c5195 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/3fa73956fc594e39b682da4b3f0c5195 2024-11-24T20:46:41,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481261396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:41,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:41,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481261396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481261396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:41,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:41,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:41,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:41,405 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/3fa73956fc594e39b682da4b3f0c5195, entries=150, sequenceid=323, filesize=12.0 K 2024-11-24T20:46:41,407 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=174.43 KB/178620 for 6ac72b5882ff154796d9a89886bf5150 in 532ms, sequenceid=323, compaction requested=false 2024-11-24T20:46:41,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:41,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:41,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-24T20:46:41,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-24T20:46:41,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-24T20:46:41,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3100 sec 2024-11-24T20:46:41,422 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.3220 sec 2024-11-24T20:46:41,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:41,676 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:46:41,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ac72b5882ff154796d9a89886bf5150 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-24T20:46:41,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/e3e8c5e3bfd94a4d9976751ce9673719 is 50, key is test_row_0/A:col10/1732481201043/Put/seqid=0 2024-11-24T20:46:41,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:41,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481261687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:41,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:41,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741906_1082 (size=14741) 2024-11-24T20:46:41,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/e3e8c5e3bfd94a4d9976751ce9673719 2024-11-24T20:46:41,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/f51ae246a44642b095328ae8c43c0e99 is 50, key is test_row_0/B:col10/1732481201043/Put/seqid=0 2024-11-24T20:46:41,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:41,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481261716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:41,717 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/8eb1ceff8d0f4f838df6b093aae97483 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/8eb1ceff8d0f4f838df6b093aae97483 2024-11-24T20:46:41,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:41,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:41,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481261716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:41,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:41,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481261716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:41,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:41,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:41,726 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ac72b5882ff154796d9a89886bf5150/C of 6ac72b5882ff154796d9a89886bf5150 into 8eb1ceff8d0f4f838df6b093aae97483(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:46:41,726 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:41,726 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., storeName=6ac72b5882ff154796d9a89886bf5150/C, priority=12, startTime=1732481200771; duration=0sec 2024-11-24T20:46:41,726 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:46:41,726 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:C 2024-11-24T20:46:41,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741907_1083 (size=12301) 2024-11-24T20:46:41,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/f51ae246a44642b095328ae8c43c0e99 2024-11-24T20:46:41,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/3a01b69db1cd40e9bd745c1eff098aac is 50, key is test_row_0/C:col10/1732481201043/Put/seqid=0 2024-11-24T20:46:41,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741908_1084 (size=12301) 2024-11-24T20:46:41,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/3a01b69db1cd40e9bd745c1eff098aac 2024-11-24T20:46:41,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/e3e8c5e3bfd94a4d9976751ce9673719 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/e3e8c5e3bfd94a4d9976751ce9673719 2024-11-24T20:46:41,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/e3e8c5e3bfd94a4d9976751ce9673719, entries=200, sequenceid=355, filesize=14.4 K 2024-11-24T20:46:41,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/f51ae246a44642b095328ae8c43c0e99 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/f51ae246a44642b095328ae8c43c0e99 2024-11-24T20:46:41,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/f51ae246a44642b095328ae8c43c0e99, entries=150, sequenceid=355, filesize=12.0 K 2024-11-24T20:46:41,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/3a01b69db1cd40e9bd745c1eff098aac as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/3a01b69db1cd40e9bd745c1eff098aac 2024-11-24T20:46:41,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/3a01b69db1cd40e9bd745c1eff098aac, entries=150, sequenceid=355, filesize=12.0 K 2024-11-24T20:46:41,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 6ac72b5882ff154796d9a89886bf5150 in 116ms, sequenceid=355, compaction requested=true 2024-11-24T20:46:41,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:41,792 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:46:41,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:46:41,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:46:41,793 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:46:41,794 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:46:41,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:46:41,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:46:41,794 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 6ac72b5882ff154796d9a89886bf5150/A is initiating minor compaction (all files) 2024-11-24T20:46:41,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:46:41,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:46:41,794 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ac72b5882ff154796d9a89886bf5150/A in TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:41,794 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/0dba1f2787ee419caa95e537c6db0f9c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/57850d210582427cb330dd82a9399ee9, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/e3e8c5e3bfd94a4d9976751ce9673719] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp, totalSize=39.1 K 2024-11-24T20:46:41,795 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dba1f2787ee419caa95e537c6db0f9c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732481198681 2024-11-24T20:46:41,796 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:46:41,796 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 6ac72b5882ff154796d9a89886bf5150/B is initiating minor compaction (all files) 2024-11-24T20:46:41,796 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ac72b5882ff154796d9a89886bf5150/B in TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:41,796 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/21e75c296d0241d49ed4f4dc92393b97, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/89cc2077ed3041819d19304122acce5e, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/f51ae246a44642b095328ae8c43c0e99] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp, totalSize=36.7 K 2024-11-24T20:46:41,796 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57850d210582427cb330dd82a9399ee9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732481199830 2024-11-24T20:46:41,797 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 21e75c296d0241d49ed4f4dc92393b97, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732481198681 2024-11-24T20:46:41,797 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3e8c5e3bfd94a4d9976751ce9673719, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732481201037 2024-11-24T20:46:41,797 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 89cc2077ed3041819d19304122acce5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732481199830 2024-11-24T20:46:41,798 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting f51ae246a44642b095328ae8c43c0e99, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732481201037 2024-11-24T20:46:41,818 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ac72b5882ff154796d9a89886bf5150#B#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:46:41,818 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/154b55bd3c6c40a09329db5f0f5b6d6b is 50, key is test_row_0/B:col10/1732481201043/Put/seqid=0 2024-11-24T20:46:41,819 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ac72b5882ff154796d9a89886bf5150#A#compaction#73 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:46:41,820 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/113730bd81244df2babfa174ae4267bc is 50, key is test_row_0/A:col10/1732481201043/Put/seqid=0 2024-11-24T20:46:41,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:41,832 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:46:41,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ac72b5882ff154796d9a89886bf5150 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T20:46:41,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/028300b6c847464b95b785b3d2ed2cfa is 50, key is test_row_0/A:col10/1732481201798/Put/seqid=0 2024-11-24T20:46:41,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741910_1086 (size=13119) 2024-11-24T20:46:41,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741909_1085 (size=13119) 2024-11-24T20:46:41,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741911_1087 (size=12301) 2024-11-24T20:46:41,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/028300b6c847464b95b785b3d2ed2cfa 2024-11-24T20:46:41,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/ae71d2b915da4ad6b35f017744182a6c is 50, key is test_row_0/B:col10/1732481201798/Put/seqid=0 2024-11-24T20:46:41,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741912_1088 (size=12301) 2024-11-24T20:46:41,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/ae71d2b915da4ad6b35f017744182a6c 2024-11-24T20:46:41,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/02ea03dcd42d43b6a0f868b025cc7861 is 50, key is test_row_0/C:col10/1732481201798/Put/seqid=0 2024-11-24T20:46:41,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741913_1089 (size=12301) 2024-11-24T20:46:41,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/02ea03dcd42d43b6a0f868b025cc7861 2024-11-24T20:46:41,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/028300b6c847464b95b785b3d2ed2cfa as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/028300b6c847464b95b785b3d2ed2cfa 2024-11-24T20:46:41,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/028300b6c847464b95b785b3d2ed2cfa, entries=150, sequenceid=367, filesize=12.0 K 2024-11-24T20:46:41,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/ae71d2b915da4ad6b35f017744182a6c as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/ae71d2b915da4ad6b35f017744182a6c 2024-11-24T20:46:41,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/ae71d2b915da4ad6b35f017744182a6c, entries=150, sequenceid=367, filesize=12.0 K 2024-11-24T20:46:41,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/02ea03dcd42d43b6a0f868b025cc7861 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/02ea03dcd42d43b6a0f868b025cc7861 2024-11-24T20:46:41,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:41,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/02ea03dcd42d43b6a0f868b025cc7861, entries=150, sequenceid=367, filesize=12.0 K 2024-11-24T20:46:41,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481261960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:41,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:41,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:41,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 6ac72b5882ff154796d9a89886bf5150 in 136ms, sequenceid=367, compaction requested=true 2024-11-24T20:46:41,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:41,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:A, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:46:41,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T20:46:41,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:B, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:46:41,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-24T20:46:41,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:46:41,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-24T20:46:42,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:42,079 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:46:42,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ac72b5882ff154796d9a89886bf5150 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-24T20:46:42,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/0f0fbbe6ddb843878f14cab73b75944f is 50, key is test_row_0/A:col10/1732481201958/Put/seqid=0 2024-11-24T20:46:42,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741914_1090 (size=14741) 2024-11-24T20:46:42,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/0f0fbbe6ddb843878f14cab73b75944f 2024-11-24T20:46:42,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/19ac40c9d84548b7807dbb73769d3665 is 50, key is test_row_0/B:col10/1732481201958/Put/seqid=0 2024-11-24T20:46:42,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741915_1091 (size=12301) 2024-11-24T20:46:42,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:42,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481262116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:42,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:42,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:42,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:42,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:42,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481262226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:42,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:42,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:42,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:42,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:42,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481262226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:42,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:42,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:42,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:42,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:42,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481262226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:42,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:42,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481262226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:42,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:42,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:42,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:42,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:42,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:42,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:42,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T20:46:42,246 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:46:42,246 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:46:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-24T20:46:42,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T20:46:42,248 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:46:42,249 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:46:42,250 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:46:42,259 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/113730bd81244df2babfa174ae4267bc as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/113730bd81244df2babfa174ae4267bc 2024-11-24T20:46:42,261 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/154b55bd3c6c40a09329db5f0f5b6d6b as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/154b55bd3c6c40a09329db5f0f5b6d6b 2024-11-24T20:46:42,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:42,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481262266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:42,267 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_0' in 6ac72b5882ff154796d9a89886bf5150 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:46:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:42,269 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ac72b5882ff154796d9a89886bf5150/A of 6ac72b5882ff154796d9a89886bf5150 into 113730bd81244df2babfa174ae4267bc(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:46:42,269 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:42,269 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., storeName=6ac72b5882ff154796d9a89886bf5150/A, priority=13, startTime=1732481201792; duration=0sec 2024-11-24T20:46:42,269 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-24T20:46:42,269 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:A 2024-11-24T20:46:42,269 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:A 2024-11-24T20:46:42,269 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:46:42,271 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:46:42,271 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 6ac72b5882ff154796d9a89886bf5150/C is initiating minor compaction (all files) 2024-11-24T20:46:42,271 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ac72b5882ff154796d9a89886bf5150/C in TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:42,271 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ac72b5882ff154796d9a89886bf5150/B of 6ac72b5882ff154796d9a89886bf5150 into 154b55bd3c6c40a09329db5f0f5b6d6b(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:46:42,271 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:42,271 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., storeName=6ac72b5882ff154796d9a89886bf5150/B, priority=13, startTime=1732481201793; duration=0sec 2024-11-24T20:46:42,271 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/8eb1ceff8d0f4f838df6b093aae97483, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/3fa73956fc594e39b682da4b3f0c5195, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/3a01b69db1cd40e9bd745c1eff098aac, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/02ea03dcd42d43b6a0f868b025cc7861] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp, totalSize=48.8 K 2024-11-24T20:46:42,272 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-24T20:46:42,272 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:B 2024-11-24T20:46:42,272 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:B 2024-11-24T20:46:42,272 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-24T20:46:42,272 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-24T20:46:42,272 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8eb1ceff8d0f4f838df6b093aae97483, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732481198681 2024-11-24T20:46:42,272 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-24T20:46:42,272 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. because compaction request was cancelled 2024-11-24T20:46:42,272 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:C 2024-11-24T20:46:42,272 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-24T20:46:42,272 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fa73956fc594e39b682da4b3f0c5195, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732481199830 2024-11-24T20:46:42,273 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-24T20:46:42,273 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-24T20:46:42,273 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a01b69db1cd40e9bd745c1eff098aac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732481201037 2024-11-24T20:46:42,273 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. because compaction request was cancelled 2024-11-24T20:46:42,273 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:B 2024-11-24T20:46:42,273 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-24T20:46:42,273 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02ea03dcd42d43b6a0f868b025cc7861, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732481201798 2024-11-24T20:46:42,274 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-24T20:46:42,274 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-24T20:46:42,275 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. because compaction request was cancelled 2024-11-24T20:46:42,275 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:A 2024-11-24T20:46:42,291 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ac72b5882ff154796d9a89886bf5150#C#compaction#79 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:46:42,292 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/f0c05a4beae140d3b0131fefb26f7550 is 50, key is test_row_0/C:col10/1732481201798/Put/seqid=0 2024-11-24T20:46:42,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741916_1092 (size=13153) 2024-11-24T20:46:42,304 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/f0c05a4beae140d3b0131fefb26f7550 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/f0c05a4beae140d3b0131fefb26f7550 2024-11-24T20:46:42,313 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ac72b5882ff154796d9a89886bf5150/C of 6ac72b5882ff154796d9a89886bf5150 into f0c05a4beae140d3b0131fefb26f7550(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:46:42,313 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:42,313 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., storeName=6ac72b5882ff154796d9a89886bf5150/C, priority=12, startTime=1732481201968; duration=0sec 2024-11-24T20:46:42,313 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:46:42,313 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:C 2024-11-24T20:46:42,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T20:46:42,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-24T20:46:42,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:42,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:42,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:42,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:42,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:42,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:42,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:42,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481262435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:42,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:42,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:42,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:42,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/19ac40c9d84548b7807dbb73769d3665 2024-11-24T20:46:42,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/228bbb4e26e84b228f6c3c5154b8ad53 is 50, key is test_row_0/C:col10/1732481201958/Put/seqid=0 2024-11-24T20:46:42,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741917_1093 (size=12301) 2024-11-24T20:46:42,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/228bbb4e26e84b228f6c3c5154b8ad53 2024-11-24T20:46:42,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/0f0fbbe6ddb843878f14cab73b75944f as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/0f0fbbe6ddb843878f14cab73b75944f 2024-11-24T20:46:42,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/0f0fbbe6ddb843878f14cab73b75944f, entries=200, sequenceid=393, filesize=14.4 K 2024-11-24T20:46:42,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/19ac40c9d84548b7807dbb73769d3665 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/19ac40c9d84548b7807dbb73769d3665 2024-11-24T20:46:42,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/19ac40c9d84548b7807dbb73769d3665, entries=150, sequenceid=393, filesize=12.0 K 2024-11-24T20:46:42,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-24T20:46:42,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:42,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:42,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:42,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:42,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:42,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/228bbb4e26e84b228f6c3c5154b8ad53 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/228bbb4e26e84b228f6c3c5154b8ad53 2024-11-24T20:46:42,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:42,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T20:46:42,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/228bbb4e26e84b228f6c3c5154b8ad53, entries=150, sequenceid=393, filesize=12.0 K 2024-11-24T20:46:42,572 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 6ac72b5882ff154796d9a89886bf5150 in 492ms, sequenceid=393, compaction requested=true 2024-11-24T20:46:42,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:42,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:46:42,573 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:46:42,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:46:42,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:46:42,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:46:42,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:46:42,573 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:46:42,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:46:42,575 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:46:42,575 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 6ac72b5882ff154796d9a89886bf5150/B is initiating minor compaction (all files) 2024-11-24T20:46:42,575 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ac72b5882ff154796d9a89886bf5150/B in TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:42,575 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/154b55bd3c6c40a09329db5f0f5b6d6b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/ae71d2b915da4ad6b35f017744182a6c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/19ac40c9d84548b7807dbb73769d3665] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp, totalSize=36.8 K 2024-11-24T20:46:42,576 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:46:42,576 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 6ac72b5882ff154796d9a89886bf5150/A is initiating minor compaction (all files) 2024-11-24T20:46:42,576 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ac72b5882ff154796d9a89886bf5150/A in TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:42,576 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/113730bd81244df2babfa174ae4267bc, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/028300b6c847464b95b785b3d2ed2cfa, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/0f0fbbe6ddb843878f14cab73b75944f] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp, totalSize=39.2 K 2024-11-24T20:46:42,576 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 154b55bd3c6c40a09329db5f0f5b6d6b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732481201037 2024-11-24T20:46:42,577 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 113730bd81244df2babfa174ae4267bc, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732481201037 2024-11-24T20:46:42,577 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 028300b6c847464b95b785b3d2ed2cfa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732481201798 2024-11-24T20:46:42,577 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting ae71d2b915da4ad6b35f017744182a6c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732481201798 2024-11-24T20:46:42,578 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f0fbbe6ddb843878f14cab73b75944f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732481201937 2024-11-24T20:46:42,578 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 19ac40c9d84548b7807dbb73769d3665, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732481201937 2024-11-24T20:46:42,592 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ac72b5882ff154796d9a89886bf5150#A#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:46:42,594 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/4c4a2b18913a4c94aec3a61b14bcd340 is 50, key is test_row_0/A:col10/1732481201958/Put/seqid=0 2024-11-24T20:46:42,602 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ac72b5882ff154796d9a89886bf5150#B#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:46:42,602 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/ffe66e3841e74d5a972103a8143124aa is 50, key is test_row_0/B:col10/1732481201958/Put/seqid=0 2024-11-24T20:46:42,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741919_1095 (size=13221) 2024-11-24T20:46:42,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741918_1094 (size=13221) 2024-11-24T20:46:42,617 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/ffe66e3841e74d5a972103a8143124aa as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/ffe66e3841e74d5a972103a8143124aa 2024-11-24T20:46:42,622 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/4c4a2b18913a4c94aec3a61b14bcd340 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/4c4a2b18913a4c94aec3a61b14bcd340 2024-11-24T20:46:42,626 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ac72b5882ff154796d9a89886bf5150/B of 6ac72b5882ff154796d9a89886bf5150 into ffe66e3841e74d5a972103a8143124aa(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:46:42,626 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:42,626 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., storeName=6ac72b5882ff154796d9a89886bf5150/B, priority=13, startTime=1732481202573; duration=0sec 2024-11-24T20:46:42,626 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:46:42,626 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:B 2024-11-24T20:46:42,627 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-24T20:46:42,627 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-24T20:46:42,628 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-24T20:46:42,628 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. because compaction request was cancelled 2024-11-24T20:46:42,628 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:C 2024-11-24T20:46:42,630 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ac72b5882ff154796d9a89886bf5150/A of 6ac72b5882ff154796d9a89886bf5150 into 4c4a2b18913a4c94aec3a61b14bcd340(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:46:42,630 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:42,630 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., storeName=6ac72b5882ff154796d9a89886bf5150/A, priority=13, startTime=1732481202573; duration=0sec 2024-11-24T20:46:42,630 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:46:42,630 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:A 2024-11-24T20:46:42,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-24T20:46:42,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:42,715 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 6ac72b5882ff154796d9a89886bf5150 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-24T20:46:42,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/20103da7482d4ff7ad3f679c0c8c97ba is 50, key is test_row_0/A:col10/1732481202114/Put/seqid=0 2024-11-24T20:46:42,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741920_1096 (size=12301) 2024-11-24T20:46:42,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:42,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:42,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:42,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481262815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:42,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:42,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:42,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:42,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T20:46:42,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:42,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481262925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:42,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:42,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:42,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:43,125 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/20103da7482d4ff7ad3f679c0c8c97ba 2024-11-24T20:46:43,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/d44128f4780b4e73a627c6c063e7e5b7 is 50, key is test_row_0/B:col10/1732481202114/Put/seqid=0 2024-11-24T20:46:43,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:43,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481263135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:43,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:43,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:43,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:43,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741921_1097 (size=12301) 2024-11-24T20:46:43,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481263236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:43,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481263236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:43,237 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_2' in 6ac72b5882ff154796d9a89886bf5150 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:46:43,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:43,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:43,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:43,238 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_0' in 6ac72b5882ff154796d9a89886bf5150 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:46:43,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:43,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:43,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:43,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:43,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481263245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:43,246 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_0' in 6ac72b5882ff154796d9a89886bf5150 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:46:43,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:43,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:43,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:43,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T20:46:43,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:43,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481263445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:43,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:43,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:43,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:43,546 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/d44128f4780b4e73a627c6c063e7e5b7 2024-11-24T20:46:43,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/ac185bba1674443e88fae7c9c07dce33 is 50, key is test_row_0/C:col10/1732481202114/Put/seqid=0 2024-11-24T20:46:43,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741922_1098 (size=12301) 2024-11-24T20:46:43,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:43,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481263955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:43,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:43,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:43,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:43,969 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/ac185bba1674443e88fae7c9c07dce33 2024-11-24T20:46:43,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/20103da7482d4ff7ad3f679c0c8c97ba as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/20103da7482d4ff7ad3f679c0c8c97ba 2024-11-24T20:46:43,981 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/20103da7482d4ff7ad3f679c0c8c97ba, entries=150, sequenceid=408, filesize=12.0 K 2024-11-24T20:46:43,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/d44128f4780b4e73a627c6c063e7e5b7 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/d44128f4780b4e73a627c6c063e7e5b7 2024-11-24T20:46:43,988 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/d44128f4780b4e73a627c6c063e7e5b7, entries=150, sequenceid=408, filesize=12.0 K 2024-11-24T20:46:43,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/ac185bba1674443e88fae7c9c07dce33 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/ac185bba1674443e88fae7c9c07dce33 2024-11-24T20:46:43,998 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/ac185bba1674443e88fae7c9c07dce33, entries=150, sequenceid=408, filesize=12.0 K 2024-11-24T20:46:43,999 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 6ac72b5882ff154796d9a89886bf5150 in 1284ms, sequenceid=408, compaction requested=true 2024-11-24T20:46:44,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:44,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:44,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-24T20:46:44,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-24T20:46:44,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-24T20:46:44,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7530 sec 2024-11-24T20:46:44,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.7630 sec 2024-11-24T20:46:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T20:46:44,395 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:46:44,395 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:46:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-24T20:46:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T20:46:44,397 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:46:44,398 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:46:44,398 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:46:44,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T20:46:44,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T20:46:44,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:44,551 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 6ac72b5882ff154796d9a89886bf5150 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-24T20:46:44,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/ca8f6f477bf54353ac2453547508e860 is 50, key is test_row_0/A:col10/1732481202796/Put/seqid=0 2024-11-24T20:46:44,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741923_1099 (size=12301) 2024-11-24T20:46:44,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T20:46:44,962 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/ca8f6f477bf54353ac2453547508e860 2024-11-24T20:46:44,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:44,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:44,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/e83769b461e94ee99013db028a9f0c96 is 50, key is test_row_0/B:col10/1732481202796/Put/seqid=0 2024-11-24T20:46:44,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741924_1100 (size=12301) 2024-11-24T20:46:44,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:44,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481264987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:44,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:44,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:44,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:45,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T20:46:45,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:45,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481265095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:45,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:45,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:45,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:45,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:45,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40682 deadline: 1732481265255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:45,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:45,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40668 deadline: 1732481265255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:45,256 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_2' in 6ac72b5882ff154796d9a89886bf5150 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:46:45,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:45,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:45,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:45,257 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_0' in 6ac72b5882ff154796d9a89886bf5150 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:46:45,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:45,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:45,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:45,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:45,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40684 deadline: 1732481265274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:45,276 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_0' in 6ac72b5882ff154796d9a89886bf5150 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:46:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:45,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:45,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481265305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:45,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:45,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:45,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:45,378 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/e83769b461e94ee99013db028a9f0c96 2024-11-24T20:46:45,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/6ff6f1c86cd548cca64030ff40c20b10 is 50, key is test_row_0/C:col10/1732481202796/Put/seqid=0 2024-11-24T20:46:45,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741925_1101 (size=12301) 2024-11-24T20:46:45,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T20:46:45,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:45,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481265614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:45,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:45,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:45,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:45,790 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/6ff6f1c86cd548cca64030ff40c20b10 2024-11-24T20:46:45,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/ca8f6f477bf54353ac2453547508e860 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/ca8f6f477bf54353ac2453547508e860 2024-11-24T20:46:45,801 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/ca8f6f477bf54353ac2453547508e860, entries=150, sequenceid=434, filesize=12.0 K 2024-11-24T20:46:45,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/e83769b461e94ee99013db028a9f0c96 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/e83769b461e94ee99013db028a9f0c96 2024-11-24T20:46:45,808 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/e83769b461e94ee99013db028a9f0c96, entries=150, sequenceid=434, filesize=12.0 K 2024-11-24T20:46:45,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/6ff6f1c86cd548cca64030ff40c20b10 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/6ff6f1c86cd548cca64030ff40c20b10 2024-11-24T20:46:45,815 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/6ff6f1c86cd548cca64030ff40c20b10, entries=150, sequenceid=434, filesize=12.0 K 2024-11-24T20:46:45,816 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 6ac72b5882ff154796d9a89886bf5150 in 1266ms, sequenceid=434, compaction requested=true 2024-11-24T20:46:45,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:45,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:45,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-24T20:46:45,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-24T20:46:45,824 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-24T20:46:45,825 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4200 sec 2024-11-24T20:46:45,828 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.4300 sec 2024-11-24T20:46:46,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:46,127 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:46:46,127 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ac72b5882ff154796d9a89886bf5150 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T20:46:46,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/bdc6c3d8ae1a4ac09f2ab8c4a4e32e7b is 50, key is test_row_0/A:col10/1732481204986/Put/seqid=0 2024-11-24T20:46:46,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741926_1102 (size=14741) 2024-11-24T20:46:46,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:46,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481266268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:46,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:46,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:46,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40700 deadline: 1732481266295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:46,298 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_0' in 6ac72b5882ff154796d9a89886bf5150 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:46:46,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:46,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:46,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:46,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:46,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481266375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:46,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:46,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:46,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:46,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/bdc6c3d8ae1a4ac09f2ab8c4a4e32e7b 2024-11-24T20:46:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T20:46:46,555 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:46:46,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/a8fc9a1709b3449296c6cf8e9cfb481f is 50, key is test_row_0/B:col10/1732481204986/Put/seqid=0 2024-11-24T20:46:46,556 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:46:46,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-24T20:46:46,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T20:46:46,558 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:46:46,558 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:46:46,559 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:46:46,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741927_1103 (size=12301) 2024-11-24T20:46:46,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:46,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481266585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:46,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:46,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:46,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:46,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T20:46:46,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-24T20:46:46,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:46,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:46,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:46,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:46,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:46,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:46,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-24T20:46:46,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:46,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:46,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:46,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:46,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:46,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:46,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T20:46:46,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:46,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481266895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:46,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:46,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:46,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:46,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/a8fc9a1709b3449296c6cf8e9cfb481f 2024-11-24T20:46:46,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/a712da1bdc624f6297a00011eb49db8d is 50, key is test_row_0/C:col10/1732481204986/Put/seqid=0 2024-11-24T20:46:46,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741928_1104 (size=12301) 2024-11-24T20:46:47,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-24T20:46:47,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:47,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:47,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:47,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:47,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:47,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:47,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-24T20:46:47,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:47,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:47,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:47,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:47,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:47,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:47,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T20:46:47,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-24T20:46:47,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:47,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. as already flushing 2024-11-24T20:46:47,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:47,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:47,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:47,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:46:47,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/a712da1bdc624f6297a00011eb49db8d 2024-11-24T20:46:47,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/bdc6c3d8ae1a4ac09f2ab8c4a4e32e7b as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/bdc6c3d8ae1a4ac09f2ab8c4a4e32e7b 2024-11-24T20:46:47,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/bdc6c3d8ae1a4ac09f2ab8c4a4e32e7b, entries=200, sequenceid=445, filesize=14.4 K 2024-11-24T20:46:47,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/a8fc9a1709b3449296c6cf8e9cfb481f as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/a8fc9a1709b3449296c6cf8e9cfb481f 2024-11-24T20:46:47,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/a8fc9a1709b3449296c6cf8e9cfb481f, entries=150, sequenceid=445, filesize=12.0 K 2024-11-24T20:46:47,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/a712da1bdc624f6297a00011eb49db8d as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/a712da1bdc624f6297a00011eb49db8d 2024-11-24T20:46:47,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/a712da1bdc624f6297a00011eb49db8d, entries=150, sequenceid=445, filesize=12.0 K 2024-11-24T20:46:47,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:46:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40652 deadline: 1732481267405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:47,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:47,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:46:47,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:46:47,406 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 6ac72b5882ff154796d9a89886bf5150 in 1279ms, sequenceid=445, compaction requested=true 2024-11-24T20:46:47,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:47,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:46:47,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:46:47,407 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:46:47,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:46:47,407 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:46:47,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:46:47,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ac72b5882ff154796d9a89886bf5150:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:46:47,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:46:47,408 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:46:47,408 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52564 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:46:47,408 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 6ac72b5882ff154796d9a89886bf5150/B is initiating minor compaction (all files) 2024-11-24T20:46:47,408 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 6ac72b5882ff154796d9a89886bf5150/A is initiating minor compaction (all files) 2024-11-24T20:46:47,408 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ac72b5882ff154796d9a89886bf5150/B in TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:47,408 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ac72b5882ff154796d9a89886bf5150/A in TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:47,408 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/ffe66e3841e74d5a972103a8143124aa, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/d44128f4780b4e73a627c6c063e7e5b7, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/e83769b461e94ee99013db028a9f0c96, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/a8fc9a1709b3449296c6cf8e9cfb481f] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp, totalSize=48.9 K 2024-11-24T20:46:47,408 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/4c4a2b18913a4c94aec3a61b14bcd340, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/20103da7482d4ff7ad3f679c0c8c97ba, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/ca8f6f477bf54353ac2453547508e860, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/bdc6c3d8ae1a4ac09f2ab8c4a4e32e7b] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp, totalSize=51.3 K 2024-11-24T20:46:47,409 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c4a2b18913a4c94aec3a61b14bcd340, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732481201937 2024-11-24T20:46:47,409 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting ffe66e3841e74d5a972103a8143124aa, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732481201937 2024-11-24T20:46:47,409 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20103da7482d4ff7ad3f679c0c8c97ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1732481202110 2024-11-24T20:46:47,409 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting d44128f4780b4e73a627c6c063e7e5b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1732481202110 2024-11-24T20:46:47,409 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca8f6f477bf54353ac2453547508e860, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732481202792 2024-11-24T20:46:47,409 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting e83769b461e94ee99013db028a9f0c96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732481202792 2024-11-24T20:46:47,410 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdc6c3d8ae1a4ac09f2ab8c4a4e32e7b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732481204975 2024-11-24T20:46:47,410 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting a8fc9a1709b3449296c6cf8e9cfb481f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732481204982 2024-11-24T20:46:47,419 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ac72b5882ff154796d9a89886bf5150#B#compaction#92 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:46:47,420 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/d5544f4695044fa1bb93c64b0de95315 is 50, key is test_row_0/B:col10/1732481204986/Put/seqid=0 2024-11-24T20:46:47,420 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ac72b5882ff154796d9a89886bf5150#A#compaction#93 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:46:47,421 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/f85bf8fd60cc49979767749871dd897f is 50, key is test_row_0/A:col10/1732481204986/Put/seqid=0 2024-11-24T20:46:47,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741929_1105 (size=13357) 2024-11-24T20:46:47,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741930_1106 (size=13357) 2024-11-24T20:46:47,434 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/d5544f4695044fa1bb93c64b0de95315 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/d5544f4695044fa1bb93c64b0de95315 2024-11-24T20:46:47,440 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ac72b5882ff154796d9a89886bf5150/B of 6ac72b5882ff154796d9a89886bf5150 into d5544f4695044fa1bb93c64b0de95315(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:46:47,440 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:47,440 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., storeName=6ac72b5882ff154796d9a89886bf5150/B, priority=12, startTime=1732481207407; duration=0sec 2024-11-24T20:46:47,440 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:46:47,440 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:B 2024-11-24T20:46:47,440 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-24T20:46:47,442 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62357 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-24T20:46:47,442 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 6ac72b5882ff154796d9a89886bf5150/C is initiating minor compaction (all files) 2024-11-24T20:46:47,443 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ac72b5882ff154796d9a89886bf5150/C in TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:47,443 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/f0c05a4beae140d3b0131fefb26f7550, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/228bbb4e26e84b228f6c3c5154b8ad53, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/ac185bba1674443e88fae7c9c07dce33, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/6ff6f1c86cd548cca64030ff40c20b10, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/a712da1bdc624f6297a00011eb49db8d] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp, totalSize=60.9 K 2024-11-24T20:46:47,443 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting f0c05a4beae140d3b0131fefb26f7550, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732481201798 2024-11-24T20:46:47,443 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 228bbb4e26e84b228f6c3c5154b8ad53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732481201937 2024-11-24T20:46:47,444 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting ac185bba1674443e88fae7c9c07dce33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1732481202110 2024-11-24T20:46:47,445 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ff6f1c86cd548cca64030ff40c20b10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732481202792 2024-11-24T20:46:47,445 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting a712da1bdc624f6297a00011eb49db8d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732481204982 2024-11-24T20:46:47,467 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ac72b5882ff154796d9a89886bf5150#C#compaction#94 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:46:47,468 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/0623611202d54352838e41457c710bac is 50, key is test_row_0/C:col10/1732481204986/Put/seqid=0 2024-11-24T20:46:47,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-24T20:46:47,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:47,478 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 6ac72b5882ff154796d9a89886bf5150 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-24T20:46:47,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/9235a5ee7ae24f22b34c2941272daa94 is 50, key is test_row_0/A:col10/1732481206265/Put/seqid=0 2024-11-24T20:46:47,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741931_1107 (size=13323) 2024-11-24T20:46:47,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741932_1108 (size=12301) 2024-11-24T20:46:47,493 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/9235a5ee7ae24f22b34c2941272daa94 2024-11-24T20:46:47,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/9bd0abb1608648eeb81a6a64faa09804 is 50, key is test_row_0/B:col10/1732481206265/Put/seqid=0 2024-11-24T20:46:47,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741933_1109 (size=12301) 2024-11-24T20:46:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T20:46:47,836 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/f85bf8fd60cc49979767749871dd897f as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/f85bf8fd60cc49979767749871dd897f 2024-11-24T20:46:47,843 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ac72b5882ff154796d9a89886bf5150/A of 6ac72b5882ff154796d9a89886bf5150 into f85bf8fd60cc49979767749871dd897f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:46:47,843 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:47,843 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., storeName=6ac72b5882ff154796d9a89886bf5150/A, priority=12, startTime=1732481207407; duration=0sec 2024-11-24T20:46:47,843 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:46:47,843 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:A 2024-11-24T20:46:47,900 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/0623611202d54352838e41457c710bac as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/0623611202d54352838e41457c710bac 2024-11-24T20:46:47,912 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 6ac72b5882ff154796d9a89886bf5150/C of 6ac72b5882ff154796d9a89886bf5150 into 0623611202d54352838e41457c710bac(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:46:47,912 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:47,912 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., storeName=6ac72b5882ff154796d9a89886bf5150/C, priority=11, startTime=1732481207407; duration=0sec 2024-11-24T20:46:47,913 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:46:47,913 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ac72b5882ff154796d9a89886bf5150:C 2024-11-24T20:46:47,915 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/9bd0abb1608648eeb81a6a64faa09804 2024-11-24T20:46:47,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/6dca63dc1b734082839dd6abf6584eb2 is 50, key is test_row_0/C:col10/1732481206265/Put/seqid=0 2024-11-24T20:46:47,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741934_1110 (size=12301) 2024-11-24T20:46:47,951 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/6dca63dc1b734082839dd6abf6584eb2 2024-11-24T20:46:47,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/9235a5ee7ae24f22b34c2941272daa94 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/9235a5ee7ae24f22b34c2941272daa94 2024-11-24T20:46:47,965 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/9235a5ee7ae24f22b34c2941272daa94, entries=150, sequenceid=471, filesize=12.0 K 2024-11-24T20:46:47,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/9bd0abb1608648eeb81a6a64faa09804 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/9bd0abb1608648eeb81a6a64faa09804 2024-11-24T20:46:47,973 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/9bd0abb1608648eeb81a6a64faa09804, entries=150, sequenceid=471, filesize=12.0 K 2024-11-24T20:46:47,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/6dca63dc1b734082839dd6abf6584eb2 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/6dca63dc1b734082839dd6abf6584eb2 2024-11-24T20:46:47,985 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/6dca63dc1b734082839dd6abf6584eb2, entries=150, sequenceid=471, filesize=12.0 K 2024-11-24T20:46:47,986 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 6ac72b5882ff154796d9a89886bf5150 in 508ms, sequenceid=471, compaction requested=false 2024-11-24T20:46:47,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 6ac72b5882ff154796d9a89886bf5150: 2024-11-24T20:46:47,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:47,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-24T20:46:47,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-24T20:46:47,992 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-24T20:46:47,992 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4300 sec 2024-11-24T20:46:47,995 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.4370 sec 2024-11-24T20:46:48,179 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-24T20:46:48,179 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:46:48,179 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-24T20:46:48,179 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:46:48,180 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-24T20:46:48,180 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:46:48,180 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-24T20:46:48,180 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:46:48,184 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:46:48,184 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:46:48,184 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:46:48,184 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:46:48,184 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:46:48,184 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:46:48,184 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:46:48,184 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:46:48,184 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:46:48,184 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:46:48,184 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:46:48,185 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:46:48,420 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-24T20:46:48,421 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:46:48,421 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:46:48,421 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:46:48,421 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:46:48,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T20:46:48,707 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:46:49,280 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-24T20:46:49,280 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:46:49,280 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:46:49,281 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:46:49,281 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:46:49,287 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-24T20:46:49,287 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:46:49,287 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:46:49,287 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:46:49,287 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:46:49,297 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-24T20:46:49,297 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:46:49,297 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:46:49,297 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:46:49,297 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:46:50,443 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T20:46:56,347 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-24T20:46:56,347 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:46:56,347 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:46:56,347 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:46:56,347 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:46:56,348 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T20:46:56,348 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 158 2024-11-24T20:46:56,348 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-24T20:46:56,348 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-24T20:46:56,348 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-24T20:46:56,348 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-24T20:46:56,348 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T20:46:56,348 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3806 2024-11-24T20:46:56,348 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3717 2024-11-24T20:46:56,348 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T20:46:56,348 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1731 2024-11-24T20:46:56,349 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5170 rows 2024-11-24T20:46:56,349 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1740 2024-11-24T20:46:56,349 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5203 rows 2024-11-24T20:46:56,349 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T20:46:56,349 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T20:46:56,349 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:46:56,349 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:46:56,349 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:46:56,353 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T20:46:56,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T20:46:56,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-24T20:46:56,362 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732481216361"}]},"ts":"1732481216361"} 2024-11-24T20:46:56,364 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T20:46:56,364 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T20:46:56,366 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T20:46:56,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ac72b5882ff154796d9a89886bf5150, UNASSIGN}] 2024-11-24T20:46:56,371 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ac72b5882ff154796d9a89886bf5150, UNASSIGN 2024-11-24T20:46:56,373 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=6ac72b5882ff154796d9a89886bf5150, regionState=CLOSING, regionLocation=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:56,375 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ac72b5882ff154796d9a89886bf5150, UNASSIGN because future has completed 2024-11-24T20:46:56,376 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T20:46:56,376 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291}] 2024-11-24T20:46:56,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-24T20:46:56,534 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:56,534 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T20:46:56,535 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 6ac72b5882ff154796d9a89886bf5150, disabling compactions & flushes 2024-11-24T20:46:56,535 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:56,535 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:56,535 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. after waiting 0 ms 2024-11-24T20:46:56,535 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:56,535 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 6ac72b5882ff154796d9a89886bf5150 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-24T20:46:56,540 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/cb976a410de44879af656802897b6162 is 50, key is test_row_0/A:col10/1732481216345/Put/seqid=0 2024-11-24T20:46:56,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741935_1111 (size=12301) 2024-11-24T20:46:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-24T20:46:56,946 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/cb976a410de44879af656802897b6162 2024-11-24T20:46:56,956 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/9b5fa09dcdbb4a55b954fd7985997d2b is 50, key is test_row_0/B:col10/1732481216345/Put/seqid=0 2024-11-24T20:46:56,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741936_1112 (size=12301) 2024-11-24T20:46:56,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-24T20:46:57,361 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/9b5fa09dcdbb4a55b954fd7985997d2b 2024-11-24T20:46:57,371 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/214da4c9f5ff4466805d9082af0110da is 50, key is test_row_0/C:col10/1732481216345/Put/seqid=0 2024-11-24T20:46:57,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741937_1113 (size=12301) 2024-11-24T20:46:57,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-24T20:46:57,776 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/214da4c9f5ff4466805d9082af0110da 2024-11-24T20:46:57,784 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/A/cb976a410de44879af656802897b6162 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/cb976a410de44879af656802897b6162 2024-11-24T20:46:57,789 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/cb976a410de44879af656802897b6162, entries=150, sequenceid=481, filesize=12.0 K 2024-11-24T20:46:57,790 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/B/9b5fa09dcdbb4a55b954fd7985997d2b as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/9b5fa09dcdbb4a55b954fd7985997d2b 2024-11-24T20:46:57,796 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/9b5fa09dcdbb4a55b954fd7985997d2b, entries=150, sequenceid=481, filesize=12.0 K 2024-11-24T20:46:57,797 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/.tmp/C/214da4c9f5ff4466805d9082af0110da as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/214da4c9f5ff4466805d9082af0110da 2024-11-24T20:46:57,802 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/214da4c9f5ff4466805d9082af0110da, entries=150, sequenceid=481, filesize=12.0 K 2024-11-24T20:46:57,803 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 6ac72b5882ff154796d9a89886bf5150 in 1268ms, sequenceid=481, compaction requested=true 2024-11-24T20:46:57,804 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/b33b7c5cb4ac41c8ad50f6a380a1f86c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/76cdb06b45674600959e55dcef63a574, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/6c117859f86f4515b1f31496a9f1e0d7, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/0bcd450ce37e4d3891580d8fd09818dc, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/f5a6e4290843488da599a213cd7f060b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/56b0ff1375354160be9d13c28ced7bc6, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/546476d909514a0191fe37f45f5556e9, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/4113f64b8d854c30b5e89dc166d23c03, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/e35c760e29394812b5c68d17da6b4dad, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/547287a95bf14a1da393ced68803e51b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/687ef162984d4a10bbc54d6acd5bdc19, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/68497b1299d14f6481837507215c8af9, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/38f7af416d334719a07b55a49aa26f1a, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/e7a7175f5fba4bcf8912f2fa2674db19, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/e1fe23248f42416886771f25e20739e2, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/b34a936810424b659068699c3554dcb1, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/f418a088736149d09d1cdd90c069f28b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/19d7bfb781c543b5a06705c8e1fe3f6d, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/2e846dcb03bc449fab36c78408d8c10f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/78a45aebe0ab45729b982da9fa0fc7f7, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/67cb65dd2bf34dd48dcbf4fff6133650, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/0dba1f2787ee419caa95e537c6db0f9c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/57850d210582427cb330dd82a9399ee9, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/e3e8c5e3bfd94a4d9976751ce9673719, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/113730bd81244df2babfa174ae4267bc, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/028300b6c847464b95b785b3d2ed2cfa, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/0f0fbbe6ddb843878f14cab73b75944f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/4c4a2b18913a4c94aec3a61b14bcd340, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/20103da7482d4ff7ad3f679c0c8c97ba, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/ca8f6f477bf54353ac2453547508e860, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/bdc6c3d8ae1a4ac09f2ab8c4a4e32e7b] to archive 2024-11-24T20:46:57,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T20:46:57,813 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/b33b7c5cb4ac41c8ad50f6a380a1f86c to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/b33b7c5cb4ac41c8ad50f6a380a1f86c 2024-11-24T20:46:57,815 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/76cdb06b45674600959e55dcef63a574 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/76cdb06b45674600959e55dcef63a574 2024-11-24T20:46:57,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/6c117859f86f4515b1f31496a9f1e0d7 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/6c117859f86f4515b1f31496a9f1e0d7 2024-11-24T20:46:57,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/0bcd450ce37e4d3891580d8fd09818dc to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/0bcd450ce37e4d3891580d8fd09818dc 2024-11-24T20:46:57,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/f5a6e4290843488da599a213cd7f060b to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/f5a6e4290843488da599a213cd7f060b 2024-11-24T20:46:57,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/56b0ff1375354160be9d13c28ced7bc6 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/56b0ff1375354160be9d13c28ced7bc6 2024-11-24T20:46:57,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/546476d909514a0191fe37f45f5556e9 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/546476d909514a0191fe37f45f5556e9 2024-11-24T20:46:57,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/4113f64b8d854c30b5e89dc166d23c03 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/4113f64b8d854c30b5e89dc166d23c03 2024-11-24T20:46:57,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/e35c760e29394812b5c68d17da6b4dad to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/e35c760e29394812b5c68d17da6b4dad 2024-11-24T20:46:57,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/547287a95bf14a1da393ced68803e51b to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/547287a95bf14a1da393ced68803e51b 2024-11-24T20:46:57,827 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/687ef162984d4a10bbc54d6acd5bdc19 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/687ef162984d4a10bbc54d6acd5bdc19 2024-11-24T20:46:57,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/68497b1299d14f6481837507215c8af9 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/68497b1299d14f6481837507215c8af9 2024-11-24T20:46:57,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/38f7af416d334719a07b55a49aa26f1a to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/38f7af416d334719a07b55a49aa26f1a 2024-11-24T20:46:57,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/e7a7175f5fba4bcf8912f2fa2674db19 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/e7a7175f5fba4bcf8912f2fa2674db19 2024-11-24T20:46:57,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/e1fe23248f42416886771f25e20739e2 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/e1fe23248f42416886771f25e20739e2 2024-11-24T20:46:57,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/b34a936810424b659068699c3554dcb1 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/b34a936810424b659068699c3554dcb1 2024-11-24T20:46:57,836 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/f418a088736149d09d1cdd90c069f28b to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/f418a088736149d09d1cdd90c069f28b 2024-11-24T20:46:57,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/19d7bfb781c543b5a06705c8e1fe3f6d to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/19d7bfb781c543b5a06705c8e1fe3f6d 2024-11-24T20:46:57,839 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/2e846dcb03bc449fab36c78408d8c10f to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/2e846dcb03bc449fab36c78408d8c10f 2024-11-24T20:46:57,840 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/78a45aebe0ab45729b982da9fa0fc7f7 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/78a45aebe0ab45729b982da9fa0fc7f7 2024-11-24T20:46:57,844 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/67cb65dd2bf34dd48dcbf4fff6133650 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/67cb65dd2bf34dd48dcbf4fff6133650 2024-11-24T20:46:57,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/0dba1f2787ee419caa95e537c6db0f9c to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/0dba1f2787ee419caa95e537c6db0f9c 2024-11-24T20:46:57,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/57850d210582427cb330dd82a9399ee9 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/57850d210582427cb330dd82a9399ee9 2024-11-24T20:46:57,849 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/e3e8c5e3bfd94a4d9976751ce9673719 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/e3e8c5e3bfd94a4d9976751ce9673719 2024-11-24T20:46:57,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/113730bd81244df2babfa174ae4267bc to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/113730bd81244df2babfa174ae4267bc 2024-11-24T20:46:57,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/028300b6c847464b95b785b3d2ed2cfa to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/028300b6c847464b95b785b3d2ed2cfa 2024-11-24T20:46:57,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/0f0fbbe6ddb843878f14cab73b75944f to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/0f0fbbe6ddb843878f14cab73b75944f 2024-11-24T20:46:57,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/4c4a2b18913a4c94aec3a61b14bcd340 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/4c4a2b18913a4c94aec3a61b14bcd340 2024-11-24T20:46:57,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/20103da7482d4ff7ad3f679c0c8c97ba to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/20103da7482d4ff7ad3f679c0c8c97ba 2024-11-24T20:46:57,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/ca8f6f477bf54353ac2453547508e860 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/ca8f6f477bf54353ac2453547508e860 2024-11-24T20:46:57,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/bdc6c3d8ae1a4ac09f2ab8c4a4e32e7b to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/bdc6c3d8ae1a4ac09f2ab8c4a4e32e7b 2024-11-24T20:46:57,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/a474b9ff974f404089203e11854c9430, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/223cc249e095465ea97c498104f9a0aa, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/2bba43e13a494a768f4aa87b6cfd1512, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/051d5b8f354c474b942c7ba563da946e, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/5e536cc8b9914d029fbe2f1586a5d9e6, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/ab6224f550a643a9a44bc03dd2b47a24, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/ee3bc25253a04a1a8c34200feaeaff36, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/2a229fcffe6043a789c57a6e1df07697, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/b71cc8f2c1ae4ba3b055094863b94bae, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/fa29ce8ab5b34baabb6c122878c274ae, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/d5ba3f1169364ee9a247aa21796c7abf, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/16b2b0207b78412b8b696d2795bf7083, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/a131b0f05d0e41389ebea646ee192d86, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/e43225079a3649699344a78fd85949d2, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/8439df660eec455ab3dd2e62fdf7bc17, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/3f75ecfa84cf475f84282b33c13380f4, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/3db2440010724faa9af33117f0f15f98, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/f3f7d2a5742b483888a72aa6636f178d, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/1190e90ebaa3453db4d913453a0f3a79, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/d83c00a7823c48db86ef1d3cf9c254d3, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/21e75c296d0241d49ed4f4dc92393b97, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/2177a3e521c741ac9ef5382dbb69cc67, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/89cc2077ed3041819d19304122acce5e, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/154b55bd3c6c40a09329db5f0f5b6d6b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/f51ae246a44642b095328ae8c43c0e99, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/ae71d2b915da4ad6b35f017744182a6c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/ffe66e3841e74d5a972103a8143124aa, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/19ac40c9d84548b7807dbb73769d3665, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/d44128f4780b4e73a627c6c063e7e5b7, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/e83769b461e94ee99013db028a9f0c96, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/a8fc9a1709b3449296c6cf8e9cfb481f] to archive 2024-11-24T20:46:57,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T20:46:57,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/a474b9ff974f404089203e11854c9430 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/a474b9ff974f404089203e11854c9430 2024-11-24T20:46:57,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/223cc249e095465ea97c498104f9a0aa to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/223cc249e095465ea97c498104f9a0aa 2024-11-24T20:46:57,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/2bba43e13a494a768f4aa87b6cfd1512 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/2bba43e13a494a768f4aa87b6cfd1512 2024-11-24T20:46:57,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/051d5b8f354c474b942c7ba563da946e to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/051d5b8f354c474b942c7ba563da946e 2024-11-24T20:46:57,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/5e536cc8b9914d029fbe2f1586a5d9e6 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/5e536cc8b9914d029fbe2f1586a5d9e6 2024-11-24T20:46:57,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/ab6224f550a643a9a44bc03dd2b47a24 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/ab6224f550a643a9a44bc03dd2b47a24 2024-11-24T20:46:57,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/ee3bc25253a04a1a8c34200feaeaff36 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/ee3bc25253a04a1a8c34200feaeaff36 2024-11-24T20:46:57,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/2a229fcffe6043a789c57a6e1df07697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/2a229fcffe6043a789c57a6e1df07697 2024-11-24T20:46:57,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/b71cc8f2c1ae4ba3b055094863b94bae to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/b71cc8f2c1ae4ba3b055094863b94bae 2024-11-24T20:46:57,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/fa29ce8ab5b34baabb6c122878c274ae to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/fa29ce8ab5b34baabb6c122878c274ae 2024-11-24T20:46:57,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/d5ba3f1169364ee9a247aa21796c7abf to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/d5ba3f1169364ee9a247aa21796c7abf 2024-11-24T20:46:57,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/16b2b0207b78412b8b696d2795bf7083 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/16b2b0207b78412b8b696d2795bf7083 2024-11-24T20:46:57,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/a131b0f05d0e41389ebea646ee192d86 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/a131b0f05d0e41389ebea646ee192d86 2024-11-24T20:46:57,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/e43225079a3649699344a78fd85949d2 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/e43225079a3649699344a78fd85949d2 2024-11-24T20:46:57,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/8439df660eec455ab3dd2e62fdf7bc17 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/8439df660eec455ab3dd2e62fdf7bc17 2024-11-24T20:46:57,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/3f75ecfa84cf475f84282b33c13380f4 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/3f75ecfa84cf475f84282b33c13380f4 2024-11-24T20:46:57,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/3db2440010724faa9af33117f0f15f98 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/3db2440010724faa9af33117f0f15f98 2024-11-24T20:46:57,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/f3f7d2a5742b483888a72aa6636f178d to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/f3f7d2a5742b483888a72aa6636f178d 2024-11-24T20:46:57,900 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/1190e90ebaa3453db4d913453a0f3a79 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/1190e90ebaa3453db4d913453a0f3a79 2024-11-24T20:46:57,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/d83c00a7823c48db86ef1d3cf9c254d3 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/d83c00a7823c48db86ef1d3cf9c254d3 2024-11-24T20:46:57,902 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/21e75c296d0241d49ed4f4dc92393b97 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/21e75c296d0241d49ed4f4dc92393b97 2024-11-24T20:46:57,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/2177a3e521c741ac9ef5382dbb69cc67 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/2177a3e521c741ac9ef5382dbb69cc67 2024-11-24T20:46:57,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/89cc2077ed3041819d19304122acce5e to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/89cc2077ed3041819d19304122acce5e 2024-11-24T20:46:57,906 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/154b55bd3c6c40a09329db5f0f5b6d6b to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/154b55bd3c6c40a09329db5f0f5b6d6b 2024-11-24T20:46:57,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/f51ae246a44642b095328ae8c43c0e99 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/f51ae246a44642b095328ae8c43c0e99 2024-11-24T20:46:57,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/ae71d2b915da4ad6b35f017744182a6c to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/ae71d2b915da4ad6b35f017744182a6c 2024-11-24T20:46:57,910 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/ffe66e3841e74d5a972103a8143124aa to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/ffe66e3841e74d5a972103a8143124aa 2024-11-24T20:46:57,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/19ac40c9d84548b7807dbb73769d3665 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/19ac40c9d84548b7807dbb73769d3665 2024-11-24T20:46:57,913 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/d44128f4780b4e73a627c6c063e7e5b7 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/d44128f4780b4e73a627c6c063e7e5b7 2024-11-24T20:46:57,915 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/e83769b461e94ee99013db028a9f0c96 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/e83769b461e94ee99013db028a9f0c96 2024-11-24T20:46:57,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/a8fc9a1709b3449296c6cf8e9cfb481f to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/a8fc9a1709b3449296c6cf8e9cfb481f 2024-11-24T20:46:57,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/b37fb7b205ef48ddbfa28b9354fd0c8f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/63857b9dff51435bb247a71277f01c76, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/5386731971ce4af99126a8c15227240b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/57de0894180e49528f1cb5cbb0b18d8e, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/4bb711251c204ff79d2d1915027a8e16, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/2e333cb0c2a5457ab3448deeade74977, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/030d3bc82f5344aa86b3189f5a62fc7d, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/3a7a1067db1f4d82bc4f144c743a9142, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/98310e8844ce4638bc14629709b63152, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/8abd57ebbb0c4a6ca919092ed5312322, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/9e613f26f0fa48b7be95635a01ffbf68, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/5c997faf8a24435ca6dabfaaa322378e, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/e2bec717732647cba205012a7653c6f2, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/2ceeafc6d89542e58275bf65c885c641, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/f36fd6a46b4044a9a98581832c33e332, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/5ae3f7f48f7d4552a13b61387ef7ed02, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/e79286b28bbc4d909d27475b51379e65, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/6b14590d0747467ca90ad7eb50e34704, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/43027813598b47c884b350d24cf7d9b0, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/968eaba0d1b94278a84ed5e72d36e5b2, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/8eb1ceff8d0f4f838df6b093aae97483, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/e48352d346024bceb7088a71d7af3ac8, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/3fa73956fc594e39b682da4b3f0c5195, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/3a01b69db1cd40e9bd745c1eff098aac, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/f0c05a4beae140d3b0131fefb26f7550, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/02ea03dcd42d43b6a0f868b025cc7861, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/228bbb4e26e84b228f6c3c5154b8ad53, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/ac185bba1674443e88fae7c9c07dce33, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/6ff6f1c86cd548cca64030ff40c20b10, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/a712da1bdc624f6297a00011eb49db8d] to archive 2024-11-24T20:46:57,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T20:46:57,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/b37fb7b205ef48ddbfa28b9354fd0c8f to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/b37fb7b205ef48ddbfa28b9354fd0c8f 2024-11-24T20:46:57,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/63857b9dff51435bb247a71277f01c76 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/63857b9dff51435bb247a71277f01c76 2024-11-24T20:46:57,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/5386731971ce4af99126a8c15227240b to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/5386731971ce4af99126a8c15227240b 2024-11-24T20:46:57,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/57de0894180e49528f1cb5cbb0b18d8e to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/57de0894180e49528f1cb5cbb0b18d8e 2024-11-24T20:46:57,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/4bb711251c204ff79d2d1915027a8e16 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/4bb711251c204ff79d2d1915027a8e16 2024-11-24T20:46:57,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/2e333cb0c2a5457ab3448deeade74977 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/2e333cb0c2a5457ab3448deeade74977 2024-11-24T20:46:57,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/030d3bc82f5344aa86b3189f5a62fc7d to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/030d3bc82f5344aa86b3189f5a62fc7d 2024-11-24T20:46:57,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/3a7a1067db1f4d82bc4f144c743a9142 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/3a7a1067db1f4d82bc4f144c743a9142 2024-11-24T20:46:57,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/98310e8844ce4638bc14629709b63152 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/98310e8844ce4638bc14629709b63152 2024-11-24T20:46:57,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/8abd57ebbb0c4a6ca919092ed5312322 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/8abd57ebbb0c4a6ca919092ed5312322 2024-11-24T20:46:57,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/9e613f26f0fa48b7be95635a01ffbf68 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/9e613f26f0fa48b7be95635a01ffbf68 2024-11-24T20:46:57,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/5c997faf8a24435ca6dabfaaa322378e to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/5c997faf8a24435ca6dabfaaa322378e 2024-11-24T20:46:57,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/e2bec717732647cba205012a7653c6f2 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/e2bec717732647cba205012a7653c6f2 2024-11-24T20:46:57,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/2ceeafc6d89542e58275bf65c885c641 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/2ceeafc6d89542e58275bf65c885c641 2024-11-24T20:46:57,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/f36fd6a46b4044a9a98581832c33e332 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/f36fd6a46b4044a9a98581832c33e332 2024-11-24T20:46:57,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/5ae3f7f48f7d4552a13b61387ef7ed02 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/5ae3f7f48f7d4552a13b61387ef7ed02 2024-11-24T20:46:57,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/e79286b28bbc4d909d27475b51379e65 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/e79286b28bbc4d909d27475b51379e65 2024-11-24T20:46:57,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/6b14590d0747467ca90ad7eb50e34704 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/6b14590d0747467ca90ad7eb50e34704 2024-11-24T20:46:57,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/43027813598b47c884b350d24cf7d9b0 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/43027813598b47c884b350d24cf7d9b0 2024-11-24T20:46:57,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/968eaba0d1b94278a84ed5e72d36e5b2 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/968eaba0d1b94278a84ed5e72d36e5b2 2024-11-24T20:46:57,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/8eb1ceff8d0f4f838df6b093aae97483 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/8eb1ceff8d0f4f838df6b093aae97483 2024-11-24T20:46:57,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/e48352d346024bceb7088a71d7af3ac8 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/e48352d346024bceb7088a71d7af3ac8 2024-11-24T20:46:57,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/3fa73956fc594e39b682da4b3f0c5195 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/3fa73956fc594e39b682da4b3f0c5195 2024-11-24T20:46:57,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/3a01b69db1cd40e9bd745c1eff098aac to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/3a01b69db1cd40e9bd745c1eff098aac 2024-11-24T20:46:57,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/f0c05a4beae140d3b0131fefb26f7550 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/f0c05a4beae140d3b0131fefb26f7550 2024-11-24T20:46:57,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/02ea03dcd42d43b6a0f868b025cc7861 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/02ea03dcd42d43b6a0f868b025cc7861 2024-11-24T20:46:57,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/228bbb4e26e84b228f6c3c5154b8ad53 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/228bbb4e26e84b228f6c3c5154b8ad53 2024-11-24T20:46:57,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/ac185bba1674443e88fae7c9c07dce33 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/ac185bba1674443e88fae7c9c07dce33 2024-11-24T20:46:57,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/6ff6f1c86cd548cca64030ff40c20b10 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/6ff6f1c86cd548cca64030ff40c20b10 2024-11-24T20:46:57,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/a712da1bdc624f6297a00011eb49db8d to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/a712da1bdc624f6297a00011eb49db8d 2024-11-24T20:46:57,968 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/recovered.edits/484.seqid, newMaxSeqId=484, maxSeqId=1 2024-11-24T20:46:57,969 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. 2024-11-24T20:46:57,969 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 6ac72b5882ff154796d9a89886bf5150: Waiting for close lock at 1732481216534Running coprocessor pre-close hooks at 1732481216534Disabling compacts and flushes for region at 1732481216534Disabling writes for close at 1732481216535 (+1 ms)Obtaining lock to block concurrent updates at 1732481216535Preparing flush snapshotting stores in 6ac72b5882ff154796d9a89886bf5150 at 1732481216535Finished memstore snapshotting TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732481216535Flushing stores of TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150. at 1732481216536 (+1 ms)Flushing 6ac72b5882ff154796d9a89886bf5150/A: creating writer at 1732481216536Flushing 6ac72b5882ff154796d9a89886bf5150/A: appending metadata at 1732481216539 (+3 ms)Flushing 6ac72b5882ff154796d9a89886bf5150/A: closing flushed file at 1732481216539Flushing 6ac72b5882ff154796d9a89886bf5150/B: creating writer at 1732481216952 (+413 ms)Flushing 6ac72b5882ff154796d9a89886bf5150/B: appending metadata at 1732481216956 (+4 ms)Flushing 6ac72b5882ff154796d9a89886bf5150/B: closing flushed file at 1732481216956Flushing 6ac72b5882ff154796d9a89886bf5150/C: creating writer at 1732481217367 (+411 ms)Flushing 6ac72b5882ff154796d9a89886bf5150/C: appending metadata at 1732481217370 (+3 ms)Flushing 6ac72b5882ff154796d9a89886bf5150/C: closing flushed file at 1732481217370Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67d52f43: reopening flushed file at 1732481217782 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1af4e9b: reopening flushed file at 1732481217789 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4bdf4915: reopening flushed file at 1732481217797 (+8 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 6ac72b5882ff154796d9a89886bf5150 in 1268ms, sequenceid=481, compaction requested=true at 1732481217803 (+6 ms)Writing region close event to WAL at 1732481217964 (+161 ms)Running coprocessor post-close hooks at 1732481217968 (+4 ms)Closed at 1732481217969 (+1 ms) 2024-11-24T20:46:57,973 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:57,973 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=6ac72b5882ff154796d9a89886bf5150, regionState=CLOSED 2024-11-24T20:46:57,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 because future has completed 2024-11-24T20:46:57,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-24T20:46:57,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 6ac72b5882ff154796d9a89886bf5150, server=20b981ef5c27,34625,1732481182291 in 1.6020 sec 2024-11-24T20:46:57,982 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-24T20:46:57,982 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ac72b5882ff154796d9a89886bf5150, UNASSIGN in 1.6100 sec 2024-11-24T20:46:57,985 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-24T20:46:57,986 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6180 sec 2024-11-24T20:46:57,987 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732481217987"}]},"ts":"1732481217987"} 2024-11-24T20:46:57,989 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T20:46:57,989 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T20:46:57,992 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6360 sec 2024-11-24T20:46:58,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-24T20:46:58,505 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:46:58,509 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T20:46:58,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:46:58,516 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:46:58,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-24T20:46:58,518 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:46:58,522 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:58,526 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A, FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B, FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C, FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/recovered.edits] 2024-11-24T20:46:58,531 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/9235a5ee7ae24f22b34c2941272daa94 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/9235a5ee7ae24f22b34c2941272daa94 2024-11-24T20:46:58,532 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/cb976a410de44879af656802897b6162 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/cb976a410de44879af656802897b6162 2024-11-24T20:46:58,534 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/f85bf8fd60cc49979767749871dd897f to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/A/f85bf8fd60cc49979767749871dd897f 2024-11-24T20:46:58,536 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/9b5fa09dcdbb4a55b954fd7985997d2b to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/9b5fa09dcdbb4a55b954fd7985997d2b 2024-11-24T20:46:58,538 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/9bd0abb1608648eeb81a6a64faa09804 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/9bd0abb1608648eeb81a6a64faa09804 2024-11-24T20:46:58,539 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/d5544f4695044fa1bb93c64b0de95315 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/B/d5544f4695044fa1bb93c64b0de95315 2024-11-24T20:46:58,541 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/0623611202d54352838e41457c710bac to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/0623611202d54352838e41457c710bac 2024-11-24T20:46:58,543 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/214da4c9f5ff4466805d9082af0110da to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/214da4c9f5ff4466805d9082af0110da 2024-11-24T20:46:58,544 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/6dca63dc1b734082839dd6abf6584eb2 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/C/6dca63dc1b734082839dd6abf6584eb2 2024-11-24T20:46:58,547 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/recovered.edits/484.seqid to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150/recovered.edits/484.seqid 2024-11-24T20:46:58,547 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/6ac72b5882ff154796d9a89886bf5150 2024-11-24T20:46:58,548 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T20:46:58,553 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:46:58,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-24T20:46:58,563 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T20:46:58,593 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T20:46:58,595 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:46:58,595 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T20:46:58,595 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732481218595"}]},"ts":"9223372036854775807"} 2024-11-24T20:46:58,600 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T20:46:58,600 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6ac72b5882ff154796d9a89886bf5150, NAME => 'TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T20:46:58,600 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T20:46:58,600 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732481218600"}]},"ts":"9223372036854775807"} 2024-11-24T20:46:58,604 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T20:46:58,605 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:46:58,606 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 95 msec 2024-11-24T20:46:58,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-24T20:46:58,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T20:46:58,626 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:46:58,636 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=237 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639242380_22 at /127.0.0.1:50512 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;20b981ef5c27:34625-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=554 (was 473) - SystemLoadAverage LEAK? -, ProcessCount=12 (was 11) - ProcessCount LEAK? -, AvailableMemoryMB=10219 (was 10790) 2024-11-24T20:46:58,645 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=237, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=554, ProcessCount=11, AvailableMemoryMB=10218 2024-11-24T20:46:58,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T20:46:58,647 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T20:46:58,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T20:46:58,649 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T20:46:58,649 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:46:58,649 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-24T20:46:58,650 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T20:46:58,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-24T20:46:58,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741938_1114 (size=959) 2024-11-24T20:46:58,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-24T20:46:58,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-24T20:46:59,060 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a 2024-11-24T20:46:59,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741939_1115 (size=53) 2024-11-24T20:46:59,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-24T20:46:59,470 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:46:59,470 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 170c4f62dfd32d7df8a14752b4c9e889, disabling compactions & flushes 2024-11-24T20:46:59,470 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:46:59,470 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:46:59,470 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. after waiting 0 ms 2024-11-24T20:46:59,470 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:46:59,470 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:46:59,470 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 170c4f62dfd32d7df8a14752b4c9e889: Waiting for close lock at 1732481219470Disabling compacts and flushes for region at 1732481219470Disabling writes for close at 1732481219470Writing region close event to WAL at 1732481219470Closed at 1732481219470 2024-11-24T20:46:59,472 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T20:46:59,473 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732481219472"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732481219472"}]},"ts":"1732481219472"} 2024-11-24T20:46:59,475 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T20:46:59,477 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T20:46:59,478 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732481219477"}]},"ts":"1732481219477"} 2024-11-24T20:46:59,480 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T20:46:59,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=170c4f62dfd32d7df8a14752b4c9e889, ASSIGN}] 2024-11-24T20:46:59,482 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=170c4f62dfd32d7df8a14752b4c9e889, ASSIGN 2024-11-24T20:46:59,482 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=170c4f62dfd32d7df8a14752b4c9e889, ASSIGN; state=OFFLINE, location=20b981ef5c27,34625,1732481182291; forceNewPlan=false, retain=false 2024-11-24T20:46:59,633 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=170c4f62dfd32d7df8a14752b4c9e889, regionState=OPENING, regionLocation=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:59,636 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=170c4f62dfd32d7df8a14752b4c9e889, ASSIGN because future has completed 2024-11-24T20:46:59,637 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291}] 2024-11-24T20:46:59,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-24T20:46:59,795 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:46:59,795 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} 2024-11-24T20:46:59,795 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:46:59,795 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:46:59,796 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:46:59,796 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:46:59,797 INFO [StoreOpener-170c4f62dfd32d7df8a14752b4c9e889-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:46:59,800 INFO [StoreOpener-170c4f62dfd32d7df8a14752b4c9e889-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 170c4f62dfd32d7df8a14752b4c9e889 columnFamilyName A 2024-11-24T20:46:59,800 DEBUG [StoreOpener-170c4f62dfd32d7df8a14752b4c9e889-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:46:59,801 INFO [StoreOpener-170c4f62dfd32d7df8a14752b4c9e889-1 {}] regionserver.HStore(327): Store=170c4f62dfd32d7df8a14752b4c9e889/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:46:59,801 INFO [StoreOpener-170c4f62dfd32d7df8a14752b4c9e889-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:46:59,802 INFO [StoreOpener-170c4f62dfd32d7df8a14752b4c9e889-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 170c4f62dfd32d7df8a14752b4c9e889 columnFamilyName B 2024-11-24T20:46:59,803 DEBUG [StoreOpener-170c4f62dfd32d7df8a14752b4c9e889-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:46:59,803 INFO [StoreOpener-170c4f62dfd32d7df8a14752b4c9e889-1 {}] regionserver.HStore(327): Store=170c4f62dfd32d7df8a14752b4c9e889/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:46:59,803 INFO [StoreOpener-170c4f62dfd32d7df8a14752b4c9e889-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:46:59,804 INFO [StoreOpener-170c4f62dfd32d7df8a14752b4c9e889-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 170c4f62dfd32d7df8a14752b4c9e889 columnFamilyName C 2024-11-24T20:46:59,804 DEBUG [StoreOpener-170c4f62dfd32d7df8a14752b4c9e889-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:46:59,805 INFO [StoreOpener-170c4f62dfd32d7df8a14752b4c9e889-1 {}] regionserver.HStore(327): Store=170c4f62dfd32d7df8a14752b4c9e889/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:46:59,805 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:46:59,806 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:46:59,806 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:46:59,807 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:46:59,807 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:46:59,808 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T20:46:59,809 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:46:59,812 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T20:46:59,812 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 170c4f62dfd32d7df8a14752b4c9e889; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61377820, jitterRate=-0.08539921045303345}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T20:46:59,812 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:46:59,813 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 170c4f62dfd32d7df8a14752b4c9e889: Running coprocessor pre-open hook at 1732481219796Writing region info on filesystem at 1732481219796Initializing all the Stores at 1732481219797 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481219797Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481219797Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481219797Cleaning up temporary data from old regions at 1732481219807 (+10 ms)Running coprocessor post-open hooks at 1732481219812 (+5 ms)Region opened successfully at 1732481219813 (+1 ms) 2024-11-24T20:46:59,814 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., pid=34, masterSystemTime=1732481219791 2024-11-24T20:46:59,816 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:46:59,816 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:46:59,817 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=170c4f62dfd32d7df8a14752b4c9e889, regionState=OPEN, openSeqNum=2, regionLocation=20b981ef5c27,34625,1732481182291 2024-11-24T20:46:59,818 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 because future has completed 2024-11-24T20:46:59,827 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-24T20:46:59,827 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 in 188 msec 2024-11-24T20:46:59,828 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-24T20:46:59,828 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=170c4f62dfd32d7df8a14752b4c9e889, ASSIGN in 347 msec 2024-11-24T20:46:59,829 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T20:46:59,829 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732481219829"}]},"ts":"1732481219829"} 2024-11-24T20:46:59,831 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T20:46:59,832 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T20:46:59,833 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1850 sec 2024-11-24T20:47:00,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-24T20:47:00,796 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:47:00,797 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6374bdcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:00,797 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:47:00,797 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:47:00,799 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:47:00,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:47:00,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:47:00,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@450a0fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:00,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:47:00,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:47:00,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:00,802 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33594, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:47:00,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37421322, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:00,804 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:47:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:47:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:47:00,807 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49424, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:47:00,808 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T20:47:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 20b981ef5c27,38339,1732481181363 2024-11-24T20:47:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3df6455a 2024-11-24T20:47:00,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T20:47:00,809 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T20:47:00,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T20:47:00,814 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T20:47:00,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-24T20:47:00,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741940_1116 (size=995) 2024-11-24T20:47:01,234 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-24T20:47:01,234 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-24T20:47:01,240 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T20:47:01,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=170c4f62dfd32d7df8a14752b4c9e889, REOPEN/MOVE}] 2024-11-24T20:47:01,250 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=170c4f62dfd32d7df8a14752b4c9e889, REOPEN/MOVE 2024-11-24T20:47:01,250 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=170c4f62dfd32d7df8a14752b4c9e889, regionState=CLOSING, regionLocation=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:01,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=170c4f62dfd32d7df8a14752b4c9e889, REOPEN/MOVE because future has completed 2024-11-24T20:47:01,253 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T20:47:01,253 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291}] 2024-11-24T20:47:01,405 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:01,405 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T20:47:01,405 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 170c4f62dfd32d7df8a14752b4c9e889, disabling compactions & flushes 2024-11-24T20:47:01,405 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:01,405 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:01,406 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. after waiting 0 ms 2024-11-24T20:47:01,406 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:01,410 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T20:47:01,410 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:01,411 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 170c4f62dfd32d7df8a14752b4c9e889: Waiting for close lock at 1732481221405Running coprocessor pre-close hooks at 1732481221405Disabling compacts and flushes for region at 1732481221405Disabling writes for close at 1732481221406 (+1 ms)Writing region close event to WAL at 1732481221406Running coprocessor post-close hooks at 1732481221410 (+4 ms)Closed at 1732481221410 2024-11-24T20:47:01,411 WARN [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: 170c4f62dfd32d7df8a14752b4c9e889 to self. 2024-11-24T20:47:01,413 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:01,414 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=170c4f62dfd32d7df8a14752b4c9e889, regionState=CLOSED 2024-11-24T20:47:01,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 because future has completed 2024-11-24T20:47:01,419 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-24T20:47:01,419 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure 170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 in 164 msec 2024-11-24T20:47:01,419 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=170c4f62dfd32d7df8a14752b4c9e889, REOPEN/MOVE; state=CLOSED, location=20b981ef5c27,34625,1732481182291; forceNewPlan=false, retain=true 2024-11-24T20:47:01,570 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=170c4f62dfd32d7df8a14752b4c9e889, regionState=OPENING, regionLocation=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:01,573 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=170c4f62dfd32d7df8a14752b4c9e889, REOPEN/MOVE because future has completed 2024-11-24T20:47:01,573 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291}] 2024-11-24T20:47:01,731 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:01,731 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} 2024-11-24T20:47:01,732 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:01,732 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:47:01,732 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:01,732 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:01,735 INFO [StoreOpener-170c4f62dfd32d7df8a14752b4c9e889-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:01,742 INFO [StoreOpener-170c4f62dfd32d7df8a14752b4c9e889-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 170c4f62dfd32d7df8a14752b4c9e889 columnFamilyName A 2024-11-24T20:47:01,744 DEBUG [StoreOpener-170c4f62dfd32d7df8a14752b4c9e889-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:01,745 INFO [StoreOpener-170c4f62dfd32d7df8a14752b4c9e889-1 {}] regionserver.HStore(327): Store=170c4f62dfd32d7df8a14752b4c9e889/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:47:01,745 INFO [StoreOpener-170c4f62dfd32d7df8a14752b4c9e889-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:01,746 INFO [StoreOpener-170c4f62dfd32d7df8a14752b4c9e889-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 170c4f62dfd32d7df8a14752b4c9e889 columnFamilyName B 2024-11-24T20:47:01,746 DEBUG [StoreOpener-170c4f62dfd32d7df8a14752b4c9e889-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:01,747 INFO [StoreOpener-170c4f62dfd32d7df8a14752b4c9e889-1 {}] regionserver.HStore(327): Store=170c4f62dfd32d7df8a14752b4c9e889/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:47:01,747 INFO [StoreOpener-170c4f62dfd32d7df8a14752b4c9e889-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:01,748 INFO [StoreOpener-170c4f62dfd32d7df8a14752b4c9e889-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 170c4f62dfd32d7df8a14752b4c9e889 columnFamilyName C 2024-11-24T20:47:01,748 DEBUG [StoreOpener-170c4f62dfd32d7df8a14752b4c9e889-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:01,748 INFO [StoreOpener-170c4f62dfd32d7df8a14752b4c9e889-1 {}] regionserver.HStore(327): Store=170c4f62dfd32d7df8a14752b4c9e889/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:47:01,749 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:01,750 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:01,750 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:01,752 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:01,752 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:01,752 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T20:47:01,754 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:01,755 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 170c4f62dfd32d7df8a14752b4c9e889; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64838818, jitterRate=-0.0338263213634491}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T20:47:01,755 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:01,757 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 170c4f62dfd32d7df8a14752b4c9e889: Running coprocessor pre-open hook at 1732481221733Writing region info on filesystem at 1732481221733Initializing all the Stores at 1732481221734 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481221734Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481221734Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481221734Cleaning up temporary data from old regions at 1732481221752 (+18 ms)Running coprocessor post-open hooks at 1732481221755 (+3 ms)Region opened successfully at 1732481221757 (+2 ms) 2024-11-24T20:47:01,758 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., pid=39, masterSystemTime=1732481221725 2024-11-24T20:47:01,760 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:01,760 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:01,760 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=170c4f62dfd32d7df8a14752b4c9e889, regionState=OPEN, openSeqNum=5, regionLocation=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:01,762 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 because future has completed 2024-11-24T20:47:01,765 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-24T20:47:01,765 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 in 190 msec 2024-11-24T20:47:01,767 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-24T20:47:01,767 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=170c4f62dfd32d7df8a14752b4c9e889, REOPEN/MOVE in 517 msec 2024-11-24T20:47:01,772 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-24T20:47:01,772 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 530 msec 2024-11-24T20:47:01,776 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 957 msec 2024-11-24T20:47:01,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-24T20:47:01,779 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:47:01,780 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e853879, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:01,780 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:47:01,780 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:47:01,781 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:47:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:47:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:47:01,782 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77281346, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:01,782 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:47:01,782 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:47:01,782 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:01,783 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33624, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:47:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26b0ecd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:01,784 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a00ea56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:01,784 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:47:01,785 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:47:01,785 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:47:01,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:47:01,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:47:01,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b24a261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:01,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:47:01,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:47:01,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:01,787 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33652, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:47:01,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@286f0a7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:01,789 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@636d090d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:01,789 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:47:01,789 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:47:01,790 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:47:01,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:47:01,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:47:01,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72f7391, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:01,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:47:01,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:47:01,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:01,792 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33676, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:47:01,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38b4fd10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:01,794 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63d8f52d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:01,794 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:47:01,795 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:47:01,796 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:47:01,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:47:01,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:47:01,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@337caebf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:01,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:47:01,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:47:01,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:01,798 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33686, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:47:01,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33fb57c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:01,799 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b0d456e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:01,799 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:47:01,800 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:47:01,800 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:47:01,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:47:01,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:47:01,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@204f0a33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:01,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:47:01,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:47:01,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:01,802 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33706, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:47:01,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@609de1f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:01,803 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27c26b5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:01,803 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:47:01,803 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:47:01,804 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:47:01,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:47:01,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:47:01,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c244b20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:01,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:47:01,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:47:01,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:01,820 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33722, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:47:01,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c2cc1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:01,821 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30f10c2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:01,821 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:47:01,822 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:47:01,823 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:47:01,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:47:01,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:47:01,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49afb0d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:01,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:47:01,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:47:01,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:01,824 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33744, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:47:01,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fd0e0eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:01,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7adec721, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:01,826 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:47:01,826 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:47:01,827 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:47:01,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:47:01,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:47:01,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e90d2cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:01,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:47:01,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:47:01,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:01,828 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33766, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:47:01,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b1266e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:01,830 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78c2fc25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:01,830 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:47:01,830 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:47:01,831 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:47:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:47:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:47:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@171abfe3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:47:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:47:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:01,833 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33774, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:47:01,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1389b68a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:01,834 DEBUG [Thread-600 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:47:01,834 DEBUG [Thread-602 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:47:01,834 DEBUG [Thread-605 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:47:01,835 DEBUG [Thread-609 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:47:01,835 DEBUG [Thread-598 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:47:01,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:47:01,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:47:01,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:47:01,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:47:01,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:47:01,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:47:01,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:47:01,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:47:01,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:47:01,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:47:01,837 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49432, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:47:01,838 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49440, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:47:01,838 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49428, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:47:01,838 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:47:01,838 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:47:01,840 DEBUG [Thread-594 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:47:01,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:47:01,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:47:01,842 DEBUG [Thread-607 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:47:01,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5] 2024-11-24T20:47:01,843 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49480, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:47:01,843 DEBUG [Thread-596 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:47:01,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5] 2024-11-24T20:47:01,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:47:01,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5] 2024-11-24T20:47:01,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:47:01,845 DEBUG [Thread-611 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:47:01,846 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:47:01,846 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:47:01,846 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:47:01,846 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49496, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:47:01,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5] 2024-11-24T20:47:01,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:47:01,849 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49498, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:47:01,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:47:01,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5] 2024-11-24T20:47:01,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5] 2024-11-24T20:47:01,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-24T20:47:01,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5] 2024-11-24T20:47:01,852 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49508, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:47:01,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-24T20:47:01,853 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:47:01,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5] 2024-11-24T20:47:01,855 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:47:01,855 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:47:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5] 2024-11-24T20:47:01,865 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:47:01,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 170c4f62dfd32d7df8a14752b4c9e889 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-24T20:47:01,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:01,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:01,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49508 deadline: 1732481281910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:01,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:01,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:01,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:01,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:01,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481281916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:01,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:01,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:01,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:01,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411246c6f6a8d6657438e92e1d582cf6da960_170c4f62dfd32d7df8a14752b4c9e889 is 50, key is test_row_0/A:col10/1732481221860/Put/seqid=0 2024-11-24T20:47:01,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732481281912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:01,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:01,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:01,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:01,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:01,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49456 deadline: 1732481281917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:01,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:01,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:01,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:01,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1732481281919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:01,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:01,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:01,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:01,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:01,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741941_1117 (size=17034) 2024-11-24T20:47:01,947 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:01,952 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411246c6f6a8d6657438e92e1d582cf6da960_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246c6f6a8d6657438e92e1d582cf6da960_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:01,953 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/9995049a8af0418bac5f3ff41bc15e43, store: [table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-24T20:47:01,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/9995049a8af0418bac5f3ff41bc15e43 is 175, key is test_row_0/A:col10/1732481221860/Put/seqid=0 2024-11-24T20:47:01,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741942_1118 (size=48139) 2024-11-24T20:47:02,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T20:47:02,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:02,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:02,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:02,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:02,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481282025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:02,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:02,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:02,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49508 deadline: 1732481282028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:02,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:02,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:02,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:02,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1732481282036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:02,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49456 deadline: 1732481282036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:02,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:02,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:02,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:02,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732481282036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:02,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:02,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-24T20:47:02,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T20:47:02,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:02,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:02,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:02,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:02,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481282235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:02,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:02,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:02,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1732481282246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:02,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:02,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:02,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49508 deadline: 1732481282246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:02,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:02,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:02,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732481282247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:02,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:02,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49456 deadline: 1732481282247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:02,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:02,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:02,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T20:47:02,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:02,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:02,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:02,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,383 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/9995049a8af0418bac5f3ff41bc15e43 2024-11-24T20:47:02,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/073cc978f2e5470b94a12fcb99656c26 is 50, key is test_row_0/B:col10/1732481221860/Put/seqid=0 2024-11-24T20:47:02,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741943_1119 (size=12001) 2024-11-24T20:47:02,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/073cc978f2e5470b94a12fcb99656c26 2024-11-24T20:47:02,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/1f2707c7acd543d8b4c7638ad4fba795 is 50, key is test_row_0/C:col10/1732481221860/Put/seqid=0 2024-11-24T20:47:02,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-24T20:47:02,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T20:47:02,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:02,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:02,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:02,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741944_1120 (size=12001) 2024-11-24T20:47:02,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/1f2707c7acd543d8b4c7638ad4fba795 2024-11-24T20:47:02,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/9995049a8af0418bac5f3ff41bc15e43 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/9995049a8af0418bac5f3ff41bc15e43 2024-11-24T20:47:02,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/9995049a8af0418bac5f3ff41bc15e43, entries=250, sequenceid=16, filesize=47.0 K 2024-11-24T20:47:02,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/073cc978f2e5470b94a12fcb99656c26 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/073cc978f2e5470b94a12fcb99656c26 2024-11-24T20:47:02,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/073cc978f2e5470b94a12fcb99656c26, entries=150, sequenceid=16, filesize=11.7 K 2024-11-24T20:47:02,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/1f2707c7acd543d8b4c7638ad4fba795 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/1f2707c7acd543d8b4c7638ad4fba795 2024-11-24T20:47:02,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/1f2707c7acd543d8b4c7638ad4fba795, entries=150, sequenceid=16, filesize=11.7 K 2024-11-24T20:47:02,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:02,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481282545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:02,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:02,547 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 170c4f62dfd32d7df8a14752b4c9e889 in 682ms, sequenceid=16, compaction requested=false 2024-11-24T20:47:02,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 170c4f62dfd32d7df8a14752b4c9e889: 2024-11-24T20:47:02,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:02,569 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:47:02,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 170c4f62dfd32d7df8a14752b4c9e889 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-24T20:47:02,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124a3ba1689380446a69f1d7bd70d32bab2_170c4f62dfd32d7df8a14752b4c9e889 is 50, key is test_row_0/A:col10/1732481222567/Put/seqid=0 2024-11-24T20:47:02,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:02,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732481282586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:02,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:02,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:02,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49508 deadline: 1732481282587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:02,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49456 deadline: 1732481282589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:02,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:02,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:02,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:02,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:02,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1732481282589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:02,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:02,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741945_1121 (size=14594) 2024-11-24T20:47:02,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T20:47:02,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:02,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:02,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:02,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:02,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:02,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49456 deadline: 1732481282705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:02,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49508 deadline: 1732481282705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:02,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:02,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:02,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732481282705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:02,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1732481282706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:02,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:02,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:02,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:02,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:02,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T20:47:02,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:02,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:02,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:02,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:02,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49508 deadline: 1732481282915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:02,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:02,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1732481282915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:02,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:02,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49456 deadline: 1732481282915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:02,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:02,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:02,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732481282915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:02,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:02,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:02,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:02,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:02,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T20:47:02,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:02,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:02,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:02,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:02,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-24T20:47:03,011 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:03,019 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124a3ba1689380446a69f1d7bd70d32bab2_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124a3ba1689380446a69f1d7bd70d32bab2_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:03,021 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/014ec1a9d9f840dc8afe47d46c5ceab9, store: [table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:03,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/014ec1a9d9f840dc8afe47d46c5ceab9 is 175, key is test_row_0/A:col10/1732481222567/Put/seqid=0 2024-11-24T20:47:03,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741946_1122 (size=39549) 2024-11-24T20:47:03,033 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/014ec1a9d9f840dc8afe47d46c5ceab9 2024-11-24T20:47:03,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/7c794d463c214167879880f41b7b2319 is 50, key is test_row_0/B:col10/1732481222567/Put/seqid=0 2024-11-24T20:47:03,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:03,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481283055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:03,056 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:03,056 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:03,056 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:03,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741947_1123 (size=12001) 2024-11-24T20:47:03,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T20:47:03,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:03,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:03,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:03,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49456 deadline: 1732481283225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:03,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:03,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1732481283226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:03,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:03,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:03,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49508 deadline: 1732481283226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:03,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:03,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:03,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:03,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:03,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:03,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:03,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:03,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732481283235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:03,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:03,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:03,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:03,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T20:47:03,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:03,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:03,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:03,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T20:47:03,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:03,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:03,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:03,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/7c794d463c214167879880f41b7b2319 2024-11-24T20:47:03,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/c049bd44e39c411dba45f02911d1dffa is 50, key is test_row_0/C:col10/1732481222567/Put/seqid=0 2024-11-24T20:47:03,513 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T20:47:03,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741948_1124 (size=12001) 2024-11-24T20:47:03,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T20:47:03,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:03,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:03,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:03,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T20:47:03,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:03,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:03,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:03,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:03,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1732481283735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:03,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:03,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49508 deadline: 1732481283735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:03,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:03,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49456 deadline: 1732481283735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:03,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:03,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:03,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:03,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:03,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:03,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:03,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:03,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:03,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:03,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:03,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732481283745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:03,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:03,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:03,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:03,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T20:47:03,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:03,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:03,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:03,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:03,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/c049bd44e39c411dba45f02911d1dffa 2024-11-24T20:47:03,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/014ec1a9d9f840dc8afe47d46c5ceab9 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/014ec1a9d9f840dc8afe47d46c5ceab9 2024-11-24T20:47:03,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/014ec1a9d9f840dc8afe47d46c5ceab9, entries=200, sequenceid=41, filesize=38.6 K 2024-11-24T20:47:03,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/7c794d463c214167879880f41b7b2319 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/7c794d463c214167879880f41b7b2319 2024-11-24T20:47:03,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/7c794d463c214167879880f41b7b2319, entries=150, sequenceid=41, filesize=11.7 K 2024-11-24T20:47:03,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/c049bd44e39c411dba45f02911d1dffa as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/c049bd44e39c411dba45f02911d1dffa 2024-11-24T20:47:03,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/c049bd44e39c411dba45f02911d1dffa, entries=150, sequenceid=41, filesize=11.7 K 2024-11-24T20:47:03,964 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 170c4f62dfd32d7df8a14752b4c9e889 in 1395ms, sequenceid=41, compaction requested=false 2024-11-24T20:47:03,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 170c4f62dfd32d7df8a14752b4c9e889: 2024-11-24T20:47:04,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-24T20:47:04,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T20:47:04,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:04,021 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 170c4f62dfd32d7df8a14752b4c9e889 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T20:47:04,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124540df9b4ae3c4931988f0b6e089aabef_170c4f62dfd32d7df8a14752b4c9e889 is 50, key is test_row_0/A:col10/1732481222587/Put/seqid=0 2024-11-24T20:47:04,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741949_1125 (size=12154) 2024-11-24T20:47:04,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:04,060 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124540df9b4ae3c4931988f0b6e089aabef_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124540df9b4ae3c4931988f0b6e089aabef_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:04,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/b3c0657b4af143388ed3a3a7d5912938, store: [table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:04,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/b3c0657b4af143388ed3a3a7d5912938 is 175, key is test_row_0/A:col10/1732481222587/Put/seqid=0 2024-11-24T20:47:04,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741950_1126 (size=30955) 2024-11-24T20:47:04,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:04,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:04,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:04,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481284184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:04,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:04,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:04,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:04,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:04,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481284295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:04,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:04,475 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/b3c0657b4af143388ed3a3a7d5912938 2024-11-24T20:47:04,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/3dcd852b4dc44289a7bcfecc80818625 is 50, key is test_row_0/B:col10/1732481222587/Put/seqid=0 2024-11-24T20:47:04,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741951_1127 (size=12001) 2024-11-24T20:47:04,499 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/3dcd852b4dc44289a7bcfecc80818625 2024-11-24T20:47:04,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/a4ce39dd8a804d85a5d9177aae7f1aa8 is 50, key is test_row_0/C:col10/1732481222587/Put/seqid=0 2024-11-24T20:47:04,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:04,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481284505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:04,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:04,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:04,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:04,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741952_1128 (size=12001) 2024-11-24T20:47:04,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:04,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49508 deadline: 1732481284745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:04,750 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_1' in 170c4f62dfd32d7df8a14752b4c9e889 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:47:04,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:04,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:04,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:04,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:04,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1732481284756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:04,757 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_0' in 170c4f62dfd32d7df8a14752b4c9e889 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:47:04,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:04,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:04,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:04,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:04,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732481284756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:04,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:04,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49456 deadline: 1732481284756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:04,760 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_2' in 170c4f62dfd32d7df8a14752b4c9e889 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:47:04,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:04,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:04,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:04,760 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_1' in 170c4f62dfd32d7df8a14752b4c9e889 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:47:04,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:04,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:04,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:04,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:04,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481284816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:04,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:04,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:04,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:04,916 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/a4ce39dd8a804d85a5d9177aae7f1aa8 2024-11-24T20:47:04,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/b3c0657b4af143388ed3a3a7d5912938 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/b3c0657b4af143388ed3a3a7d5912938 2024-11-24T20:47:04,927 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/b3c0657b4af143388ed3a3a7d5912938, entries=150, sequenceid=52, filesize=30.2 K 2024-11-24T20:47:04,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/3dcd852b4dc44289a7bcfecc80818625 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/3dcd852b4dc44289a7bcfecc80818625 2024-11-24T20:47:04,935 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/3dcd852b4dc44289a7bcfecc80818625, entries=150, sequenceid=52, filesize=11.7 K 2024-11-24T20:47:04,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/a4ce39dd8a804d85a5d9177aae7f1aa8 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/a4ce39dd8a804d85a5d9177aae7f1aa8 2024-11-24T20:47:04,945 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/a4ce39dd8a804d85a5d9177aae7f1aa8, entries=150, sequenceid=52, filesize=11.7 K 2024-11-24T20:47:04,947 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 170c4f62dfd32d7df8a14752b4c9e889 in 926ms, sequenceid=52, compaction requested=true 2024-11-24T20:47:04,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 170c4f62dfd32d7df8a14752b4c9e889: 2024-11-24T20:47:04,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:04,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-24T20:47:04,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-24T20:47:04,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-24T20:47:04,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0940 sec 2024-11-24T20:47:04,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 3.1060 sec 2024-11-24T20:47:05,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:05,327 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:47:05,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 170c4f62dfd32d7df8a14752b4c9e889 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-24T20:47:05,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112446adc57f1edb4fabb4b1944c58fb8918_170c4f62dfd32d7df8a14752b4c9e889 is 50, key is test_row_0/A:col10/1732481225326/Put/seqid=0 2024-11-24T20:47:05,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741953_1129 (size=14594) 2024-11-24T20:47:05,389 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,396 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112446adc57f1edb4fabb4b1944c58fb8918_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112446adc57f1edb4fabb4b1944c58fb8918_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:05,398 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/d81a0b1ca2344113840c20b06dc6e381, store: [table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:05,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/d81a0b1ca2344113840c20b06dc6e381 is 175, key is test_row_0/A:col10/1732481225326/Put/seqid=0 2024-11-24T20:47:05,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:05,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481285393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:05,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:05,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:05,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:05,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741954_1130 (size=39549) 2024-11-24T20:47:05,414 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/d81a0b1ca2344113840c20b06dc6e381 2024-11-24T20:47:05,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/fb350b5b5eb64da2bcca7585fad7fb27 is 50, key is test_row_0/B:col10/1732481225326/Put/seqid=0 2024-11-24T20:47:05,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741955_1131 (size=12001) 2024-11-24T20:47:05,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/fb350b5b5eb64da2bcca7585fad7fb27 2024-11-24T20:47:05,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/66099b6d42f44f0699441014ecbfdceb is 50, key is test_row_0/C:col10/1732481225326/Put/seqid=0 2024-11-24T20:47:05,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741956_1132 (size=12001) 2024-11-24T20:47:05,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:05,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481285505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:05,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:05,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:05,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:05,665 INFO [master/20b981ef5c27:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-24T20:47:05,665 INFO [master/20b981ef5c27:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-24T20:47:05,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481285715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:05,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:05,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:05,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:05,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/66099b6d42f44f0699441014ecbfdceb 2024-11-24T20:47:05,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/d81a0b1ca2344113840c20b06dc6e381 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/d81a0b1ca2344113840c20b06dc6e381 2024-11-24T20:47:05,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/d81a0b1ca2344113840c20b06dc6e381, entries=200, sequenceid=78, filesize=38.6 K 2024-11-24T20:47:05,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/fb350b5b5eb64da2bcca7585fad7fb27 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/fb350b5b5eb64da2bcca7585fad7fb27 2024-11-24T20:47:05,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/fb350b5b5eb64da2bcca7585fad7fb27, entries=150, sequenceid=78, filesize=11.7 K 2024-11-24T20:47:05,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/66099b6d42f44f0699441014ecbfdceb as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/66099b6d42f44f0699441014ecbfdceb 2024-11-24T20:47:05,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/66099b6d42f44f0699441014ecbfdceb, entries=150, sequenceid=78, filesize=11.7 K 2024-11-24T20:47:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,942 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 170c4f62dfd32d7df8a14752b4c9e889 in 614ms, sequenceid=78, compaction requested=true 2024-11-24T20:47:05,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 170c4f62dfd32d7df8a14752b4c9e889: 2024-11-24T20:47:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,942 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:47:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 170c4f62dfd32d7df8a14752b4c9e889:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:47:05,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:05,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 170c4f62dfd32d7df8a14752b4c9e889:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:47:05,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,943 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:47:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 170c4f62dfd32d7df8a14752b4c9e889:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:47:05,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:47:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,946 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 158192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:47:05,946 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:47:05,946 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 170c4f62dfd32d7df8a14752b4c9e889/A is initiating minor compaction (all files) 2024-11-24T20:47:05,947 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 170c4f62dfd32d7df8a14752b4c9e889/B is initiating minor compaction (all files) 2024-11-24T20:47:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,947 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 170c4f62dfd32d7df8a14752b4c9e889/A in TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:05,947 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 170c4f62dfd32d7df8a14752b4c9e889/B in TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:05,947 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/073cc978f2e5470b94a12fcb99656c26, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/7c794d463c214167879880f41b7b2319, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/3dcd852b4dc44289a7bcfecc80818625, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/fb350b5b5eb64da2bcca7585fad7fb27] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp, totalSize=46.9 K 2024-11-24T20:47:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,947 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/9995049a8af0418bac5f3ff41bc15e43, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/014ec1a9d9f840dc8afe47d46c5ceab9, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/b3c0657b4af143388ed3a3a7d5912938, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/d81a0b1ca2344113840c20b06dc6e381] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp, totalSize=154.5 K 2024-11-24T20:47:05,947 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:05,947 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. files: [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/9995049a8af0418bac5f3ff41bc15e43, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/014ec1a9d9f840dc8afe47d46c5ceab9, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/b3c0657b4af143388ed3a3a7d5912938, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/d81a0b1ca2344113840c20b06dc6e381] 2024-11-24T20:47:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,948 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 073cc978f2e5470b94a12fcb99656c26, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732481221853 2024-11-24T20:47:05,948 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9995049a8af0418bac5f3ff41bc15e43, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732481221853 2024-11-24T20:47:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,948 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c794d463c214167879880f41b7b2319, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732481221908 2024-11-24T20:47:05,949 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dcd852b4dc44289a7bcfecc80818625, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732481222585 2024-11-24T20:47:05,949 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 014ec1a9d9f840dc8afe47d46c5ceab9, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732481221908 2024-11-24T20:47:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,950 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting fb350b5b5eb64da2bcca7585fad7fb27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732481224175 2024-11-24T20:47:05,950 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3c0657b4af143388ed3a3a7d5912938, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732481222585 2024-11-24T20:47:05,950 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting d81a0b1ca2344113840c20b06dc6e381, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732481224169 2024-11-24T20:47:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,968 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,973 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 170c4f62dfd32d7df8a14752b4c9e889#B#compaction#114 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,974 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/2eb733e6c6c34c879b22af1e097adcd3 is 50, key is test_row_0/B:col10/1732481225326/Put/seqid=0 2024-11-24T20:47:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,975 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124e56afd0092914a1090e218736d391774_170c4f62dfd32d7df8a14752b4c9e889 store=[table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,980 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124e56afd0092914a1090e218736d391774_170c4f62dfd32d7df8a14752b4c9e889, store=[table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,980 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e56afd0092914a1090e218736d391774_170c4f62dfd32d7df8a14752b4c9e889 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741957_1133 (size=12139) 2024-11-24T20:47:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741958_1134 (size=4469) 2024-11-24T20:47:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,011 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 170c4f62dfd32d7df8a14752b4c9e889#A#compaction#113 average throughput is 0.57 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:06,011 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/afea27a9461b4b238744cf245fed946f is 175, key is test_row_0/A:col10/1732481225326/Put/seqid=0 2024-11-24T20:47:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,019 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/2eb733e6c6c34c879b22af1e097adcd3 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/2eb733e6c6c34c879b22af1e097adcd3 2024-11-24T20:47:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-24T20:47:06,026 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:47:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,026 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:47:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-24T20:47:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,029 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 170c4f62dfd32d7df8a14752b4c9e889/B of 170c4f62dfd32d7df8a14752b4c9e889 into 2eb733e6c6c34c879b22af1e097adcd3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:06,029 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:47:06,029 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 170c4f62dfd32d7df8a14752b4c9e889: 2024-11-24T20:47:06,029 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., storeName=170c4f62dfd32d7df8a14752b4c9e889/B, priority=12, startTime=1732481225943; duration=0sec 2024-11-24T20:47:06,029 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:47:06,029 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 170c4f62dfd32d7df8a14752b4c9e889:B 2024-11-24T20:47:06,029 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:47:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T20:47:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,030 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:47:06,030 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:47:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,031 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:47:06,031 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 170c4f62dfd32d7df8a14752b4c9e889/C is initiating minor compaction (all files) 2024-11-24T20:47:06,032 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 170c4f62dfd32d7df8a14752b4c9e889/C in TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:06,032 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/1f2707c7acd543d8b4c7638ad4fba795, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/c049bd44e39c411dba45f02911d1dffa, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/a4ce39dd8a804d85a5d9177aae7f1aa8, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/66099b6d42f44f0699441014ecbfdceb] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp, totalSize=46.9 K 2024-11-24T20:47:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,032 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f2707c7acd543d8b4c7638ad4fba795, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732481221853 2024-11-24T20:47:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,033 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting c049bd44e39c411dba45f02911d1dffa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732481221908 2024-11-24T20:47:06,033 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting a4ce39dd8a804d85a5d9177aae7f1aa8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732481222585 2024-11-24T20:47:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,034 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 66099b6d42f44f0699441014ecbfdceb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732481224175 2024-11-24T20:47:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741959_1135 (size=31093) 2024-11-24T20:47:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,042 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/afea27a9461b4b238744cf245fed946f as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/afea27a9461b4b238744cf245fed946f 2024-11-24T20:47:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,050 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 170c4f62dfd32d7df8a14752b4c9e889/A of 170c4f62dfd32d7df8a14752b4c9e889 into afea27a9461b4b238744cf245fed946f(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:06,050 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 170c4f62dfd32d7df8a14752b4c9e889: 2024-11-24T20:47:06,050 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., storeName=170c4f62dfd32d7df8a14752b4c9e889/A, priority=12, startTime=1732481225942; duration=0sec 2024-11-24T20:47:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,050 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:06,050 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 170c4f62dfd32d7df8a14752b4c9e889:A 2024-11-24T20:47:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,056 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 170c4f62dfd32d7df8a14752b4c9e889#C#compaction#115 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,057 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/1fb65818a076478e80f53807e56d460c is 50, key is test_row_0/C:col10/1732481225326/Put/seqid=0 2024-11-24T20:47:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741960_1136 (size=12139) 2024-11-24T20:47:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:06,074 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:47:06,075 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 170c4f62dfd32d7df8a14752b4c9e889 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T20:47:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411245f6cc7caaef143ea92a8dda0b9bd4bd4_170c4f62dfd32d7df8a14752b4c9e889 is 50, key is test_row_0/A:col10/1732481225391/Put/seqid=0 2024-11-24T20:47:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741961_1137 (size=12154) 2024-11-24T20:47:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,109 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,115 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411245f6cc7caaef143ea92a8dda0b9bd4bd4_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245f6cc7caaef143ea92a8dda0b9bd4bd4_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,117 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/3fc0a8a45f9e48788e303a7b3f442d83, store: [table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/3fc0a8a45f9e48788e303a7b3f442d83 is 175, key is test_row_0/A:col10/1732481225391/Put/seqid=0 2024-11-24T20:47:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741962_1138 (size=30951) 2024-11-24T20:47:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,127 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=91, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/3fc0a8a45f9e48788e303a7b3f442d83 2024-11-24T20:47:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T20:47:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/b6d59cda61ca41ada27c76565e152153 is 50, key is test_row_0/B:col10/1732481225391/Put/seqid=0 2024-11-24T20:47:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741963_1139 (size=9657) 2024-11-24T20:47:06,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/b6d59cda61ca41ada27c76565e152153 2024-11-24T20:47:06,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/e0b21b2505694fa7b1fa221256087aab is 50, key is test_row_0/C:col10/1732481225391/Put/seqid=0 2024-11-24T20:47:06,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-24T20:47:06,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:06,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:06,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:06,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741964_1140 (size=9657) 2024-11-24T20:47:06,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481286220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:06,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:06,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:06,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:06,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481286335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:06,337 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:06,337 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:06,337 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:06,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-24T20:47:06,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:06,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:06,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:06,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T20:47:06,469 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/1fb65818a076478e80f53807e56d460c as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/1fb65818a076478e80f53807e56d460c 2024-11-24T20:47:06,475 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 170c4f62dfd32d7df8a14752b4c9e889/C of 170c4f62dfd32d7df8a14752b4c9e889 into 1fb65818a076478e80f53807e56d460c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:06,475 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 170c4f62dfd32d7df8a14752b4c9e889: 2024-11-24T20:47:06,475 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., storeName=170c4f62dfd32d7df8a14752b4c9e889/C, priority=12, startTime=1732481225943; duration=0sec 2024-11-24T20:47:06,476 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:06,476 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 170c4f62dfd32d7df8a14752b4c9e889:C 2024-11-24T20:47:06,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-24T20:47:06,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:06,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:06,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:06,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:06,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:06,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481286544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:06,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:06,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:06,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:06,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/e0b21b2505694fa7b1fa221256087aab 2024-11-24T20:47:06,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/3fc0a8a45f9e48788e303a7b3f442d83 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/3fc0a8a45f9e48788e303a7b3f442d83 2024-11-24T20:47:06,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/3fc0a8a45f9e48788e303a7b3f442d83, entries=150, sequenceid=91, filesize=30.2 K 2024-11-24T20:47:06,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/b6d59cda61ca41ada27c76565e152153 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/b6d59cda61ca41ada27c76565e152153 2024-11-24T20:47:06,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/b6d59cda61ca41ada27c76565e152153, entries=100, sequenceid=91, filesize=9.4 K 2024-11-24T20:47:06,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/e0b21b2505694fa7b1fa221256087aab as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/e0b21b2505694fa7b1fa221256087aab 2024-11-24T20:47:06,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/e0b21b2505694fa7b1fa221256087aab, entries=100, sequenceid=91, filesize=9.4 K 2024-11-24T20:47:06,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 170c4f62dfd32d7df8a14752b4c9e889 in 547ms, sequenceid=91, compaction requested=false 2024-11-24T20:47:06,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 170c4f62dfd32d7df8a14752b4c9e889: 2024-11-24T20:47:06,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-24T20:47:06,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:06,646 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 170c4f62dfd32d7df8a14752b4c9e889 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-24T20:47:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T20:47:06,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244f20bfbebb954128bf211c58e04e8e40_170c4f62dfd32d7df8a14752b4c9e889 is 50, key is test_row_0/A:col10/1732481226208/Put/seqid=0 2024-11-24T20:47:06,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741965_1141 (size=12154) 2024-11-24T20:47:06,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:06,669 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244f20bfbebb954128bf211c58e04e8e40_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244f20bfbebb954128bf211c58e04e8e40_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:06,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/8fa838d4a7924c3caa656a64099e794f, store: [table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:06,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/8fa838d4a7924c3caa656a64099e794f is 175, key is test_row_0/A:col10/1732481226208/Put/seqid=0 2024-11-24T20:47:06,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741966_1142 (size=30955) 2024-11-24T20:47:06,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:06,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:06,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:06,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49508 deadline: 1732481286781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:06,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:06,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:06,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:06,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:06,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1732481286782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:06,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:06,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:06,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:06,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:06,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49456 deadline: 1732481286783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:06,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:06,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:06,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:06,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:06,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732481286785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:06,788 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_2' in 170c4f62dfd32d7df8a14752b4c9e889 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:47:06,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:06,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:06,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:06,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:06,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481286855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:06,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:06,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:06,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:06,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:06,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49508 deadline: 1732481286895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:06,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:06,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:06,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:06,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:06,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:06,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1732481286895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:06,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49456 deadline: 1732481286895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:06,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:06,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:06,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:06,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:06,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:06,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:07,080 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/8fa838d4a7924c3caa656a64099e794f 2024-11-24T20:47:07,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/f73e78ac1d2c49af9f52070b70967af0 is 50, key is test_row_0/B:col10/1732481226208/Put/seqid=0 2024-11-24T20:47:07,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741967_1143 (size=12001) 2024-11-24T20:47:07,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49508 deadline: 1732481287105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:07,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:07,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:07,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:07,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1732481287106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:07,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:07,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:07,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:07,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49456 deadline: 1732481287106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:07,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:07,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:07,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T20:47:07,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:07,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481287365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:07,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:07,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:07,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:07,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:07,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49508 deadline: 1732481287415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:07,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:07,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49456 deadline: 1732481287415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:07,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:07,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:07,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:07,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:07,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1732481287415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:07,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:07,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:07,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:07,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:07,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:07,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:07,494 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/f73e78ac1d2c49af9f52070b70967af0 2024-11-24T20:47:07,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/91dcfd6ad3d44f5b9c522bf7b55eef25 is 50, key is test_row_0/C:col10/1732481226208/Put/seqid=0 2024-11-24T20:47:07,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741968_1144 (size=12001) 2024-11-24T20:47:07,926 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/91dcfd6ad3d44f5b9c522bf7b55eef25 2024-11-24T20:47:07,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:07,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49456 deadline: 1732481287925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:07,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:07,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49508 deadline: 1732481287925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:07,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:07,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:07,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:07,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:07,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:07,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:07,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/8fa838d4a7924c3caa656a64099e794f as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/8fa838d4a7924c3caa656a64099e794f 2024-11-24T20:47:07,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:07,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1732481287935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:07,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:07,939 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/8fa838d4a7924c3caa656a64099e794f, entries=150, sequenceid=117, filesize=30.2 K 2024-11-24T20:47:07,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:07,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:07,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/f73e78ac1d2c49af9f52070b70967af0 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/f73e78ac1d2c49af9f52070b70967af0 2024-11-24T20:47:07,945 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/f73e78ac1d2c49af9f52070b70967af0, entries=150, sequenceid=117, filesize=11.7 K 2024-11-24T20:47:07,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/91dcfd6ad3d44f5b9c522bf7b55eef25 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/91dcfd6ad3d44f5b9c522bf7b55eef25 2024-11-24T20:47:07,952 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/91dcfd6ad3d44f5b9c522bf7b55eef25, entries=150, sequenceid=117, filesize=11.7 K 2024-11-24T20:47:07,953 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 170c4f62dfd32d7df8a14752b4c9e889 in 1307ms, sequenceid=117, compaction requested=true 2024-11-24T20:47:07,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 170c4f62dfd32d7df8a14752b4c9e889: 2024-11-24T20:47:07,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:07,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-24T20:47:07,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-24T20:47:07,959 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-24T20:47:07,959 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9250 sec 2024-11-24T20:47:07,961 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.9330 sec 2024-11-24T20:47:08,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T20:47:08,176 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:47:08,176 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:47:08,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-24T20:47:08,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T20:47:08,178 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:47:08,180 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:47:08,180 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:47:08,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T20:47:08,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-24T20:47:08,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:08,333 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 170c4f62dfd32d7df8a14752b4c9e889 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T20:47:08,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f2dc7f0b80be42f3816594ce653b1364_170c4f62dfd32d7df8a14752b4c9e889 is 50, key is test_row_0/A:col10/1732481226780/Put/seqid=0 2024-11-24T20:47:08,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741969_1145 (size=12154) 2024-11-24T20:47:08,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:08,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:08,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:08,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481288483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:08,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:08,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:08,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:08,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T20:47:08,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:08,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481288595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:08,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:08,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:08,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:08,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:08,755 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f2dc7f0b80be42f3816594ce653b1364_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f2dc7f0b80be42f3816594ce653b1364_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:08,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/61505695631d4a12a89ce20a456f1c56, store: [table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:08,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/61505695631d4a12a89ce20a456f1c56 is 175, key is test_row_0/A:col10/1732481226780/Put/seqid=0 2024-11-24T20:47:08,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741970_1146 (size=30955) 2024-11-24T20:47:08,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T20:47:08,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:08,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481288806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:08,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:08,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:08,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:08,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49456 deadline: 1732481288935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:08,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49508 deadline: 1732481288935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:08,941 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_2' in 170c4f62dfd32d7df8a14752b4c9e889 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:47:08,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:08,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:08,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:08,941 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_1' in 170c4f62dfd32d7df8a14752b4c9e889 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:47:08,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:08,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:08,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:08,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1732481288945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:08,946 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_1' in 170c4f62dfd32d7df8a14752b4c9e889 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:47:08,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:08,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:08,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:09,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481289115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:09,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:09,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:09,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:09,164 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=128, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/61505695631d4a12a89ce20a456f1c56 2024-11-24T20:47:09,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/3887e781204b46bc840bf0e93fb81799 is 50, key is test_row_0/B:col10/1732481226780/Put/seqid=0 2024-11-24T20:47:09,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741971_1147 (size=12001) 2024-11-24T20:47:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T20:47:09,594 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/3887e781204b46bc840bf0e93fb81799 2024-11-24T20:47:09,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/c79e80d624c846c98200f3eb980574fb is 50, key is test_row_0/C:col10/1732481226780/Put/seqid=0 2024-11-24T20:47:09,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741972_1148 (size=12001) 2024-11-24T20:47:09,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:09,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481289635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:09,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:09,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:09,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:10,008 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/c79e80d624c846c98200f3eb980574fb 2024-11-24T20:47:10,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/61505695631d4a12a89ce20a456f1c56 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/61505695631d4a12a89ce20a456f1c56 2024-11-24T20:47:10,024 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/61505695631d4a12a89ce20a456f1c56, entries=150, sequenceid=128, filesize=30.2 K 2024-11-24T20:47:10,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/3887e781204b46bc840bf0e93fb81799 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/3887e781204b46bc840bf0e93fb81799 2024-11-24T20:47:10,031 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/3887e781204b46bc840bf0e93fb81799, entries=150, sequenceid=128, filesize=11.7 K 2024-11-24T20:47:10,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/c79e80d624c846c98200f3eb980574fb as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/c79e80d624c846c98200f3eb980574fb 2024-11-24T20:47:10,039 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/c79e80d624c846c98200f3eb980574fb, entries=150, sequenceid=128, filesize=11.7 K 2024-11-24T20:47:10,040 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 170c4f62dfd32d7df8a14752b4c9e889 in 1707ms, sequenceid=128, compaction requested=true 2024-11-24T20:47:10,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 170c4f62dfd32d7df8a14752b4c9e889: 2024-11-24T20:47:10,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:10,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-24T20:47:10,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-24T20:47:10,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-24T20:47:10,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8620 sec 2024-11-24T20:47:10,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.8690 sec 2024-11-24T20:47:10,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T20:47:10,325 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:47:10,326 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:47:10,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-24T20:47:10,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T20:47:10,327 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:47:10,328 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:47:10,328 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:47:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T20:47:10,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T20:47:10,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:10,480 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 170c4f62dfd32d7df8a14752b4c9e889 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-24T20:47:10,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411242e6dcc85d8a343faa13141918a5b051d_170c4f62dfd32d7df8a14752b4c9e889 is 50, key is test_row_0/A:col10/1732481228451/Put/seqid=0 2024-11-24T20:47:10,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741973_1149 (size=12304) 2024-11-24T20:47:10,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T20:47:10,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:10,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:10,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:10,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481290679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:10,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:10,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481290795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:10,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:10,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:10,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:10,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:10,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732481290805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:10,807 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_2' in 170c4f62dfd32d7df8a14752b4c9e889 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:47:10,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:10,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:10,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:10,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:10,907 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411242e6dcc85d8a343faa13141918a5b051d_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242e6dcc85d8a343faa13141918a5b051d_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:10,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/40b6f82ef8d94a9cb351e7f518d5b0ab, store: [table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:10,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/40b6f82ef8d94a9cb351e7f518d5b0ab is 175, key is test_row_0/A:col10/1732481228451/Put/seqid=0 2024-11-24T20:47:10,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741974_1150 (size=31105) 2024-11-24T20:47:10,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T20:47:10,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:10,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49456 deadline: 1732481290955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:10,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:10,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49508 deadline: 1732481290956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:10,959 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_2' in 170c4f62dfd32d7df8a14752b4c9e889 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:47:10,960 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_1' in 170c4f62dfd32d7df8a14752b4c9e889 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:47:10,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:10,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:10,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:10,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:10,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:10,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:10,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:10,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1732481290965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:10,969 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_1' in 170c4f62dfd32d7df8a14752b4c9e889 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:47:10,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:11,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:11,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481291005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:11,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:11,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:11,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:11,314 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/40b6f82ef8d94a9cb351e7f518d5b0ab 2024-11-24T20:47:11,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:11,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481291315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:11,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:11,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:11,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:11,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/73e52585e63d4cbd84093d2a9d5353be is 50, key is test_row_0/B:col10/1732481228451/Put/seqid=0 2024-11-24T20:47:11,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741975_1151 (size=12151) 2024-11-24T20:47:11,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T20:47:11,731 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/73e52585e63d4cbd84093d2a9d5353be 2024-11-24T20:47:11,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/bf7eec729d494bc2b64429575d46abf3 is 50, key is test_row_0/C:col10/1732481228451/Put/seqid=0 2024-11-24T20:47:11,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741976_1152 (size=12151) 2024-11-24T20:47:11,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:11,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481291825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:11,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:12,154 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/bf7eec729d494bc2b64429575d46abf3 2024-11-24T20:47:12,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/40b6f82ef8d94a9cb351e7f518d5b0ab as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/40b6f82ef8d94a9cb351e7f518d5b0ab 2024-11-24T20:47:12,165 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/40b6f82ef8d94a9cb351e7f518d5b0ab, entries=150, sequenceid=153, filesize=30.4 K 2024-11-24T20:47:12,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/73e52585e63d4cbd84093d2a9d5353be as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/73e52585e63d4cbd84093d2a9d5353be 2024-11-24T20:47:12,172 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/73e52585e63d4cbd84093d2a9d5353be, entries=150, sequenceid=153, filesize=11.9 K 2024-11-24T20:47:12,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/bf7eec729d494bc2b64429575d46abf3 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/bf7eec729d494bc2b64429575d46abf3 2024-11-24T20:47:12,179 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/bf7eec729d494bc2b64429575d46abf3, entries=150, sequenceid=153, filesize=11.9 K 2024-11-24T20:47:12,181 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 170c4f62dfd32d7df8a14752b4c9e889 in 1700ms, sequenceid=153, compaction requested=true 2024-11-24T20:47:12,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 170c4f62dfd32d7df8a14752b4c9e889: 2024-11-24T20:47:12,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:12,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-24T20:47:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-24T20:47:12,185 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-24T20:47:12,185 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8550 sec 2024-11-24T20:47:12,188 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.8600 sec 2024-11-24T20:47:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T20:47:12,485 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:47:12,485 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:47:12,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-24T20:47:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T20:47:12,488 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:47:12,489 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:47:12,489 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:47:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T20:47:12,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T20:47:12,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:12,643 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 170c4f62dfd32d7df8a14752b4c9e889 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T20:47:12,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124107924c712a949419a4c07ab1a7bfedd_170c4f62dfd32d7df8a14752b4c9e889 is 50, key is test_row_0/A:col10/1732481230667/Put/seqid=0 2024-11-24T20:47:12,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741977_1153 (size=12304) 2024-11-24T20:47:12,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T20:47:12,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:12,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:12,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:12,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481292958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:12,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:12,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:12,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:13,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:13,064 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124107924c712a949419a4c07ab1a7bfedd_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124107924c712a949419a4c07ab1a7bfedd_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:13,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/95075228b1e346cca3297a0befa451a0, store: [table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:13,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/95075228b1e346cca3297a0befa451a0 is 175, key is test_row_0/A:col10/1732481230667/Put/seqid=0 2024-11-24T20:47:13,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741978_1154 (size=31105) 2024-11-24T20:47:13,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:13,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481293075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:13,085 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:13,085 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:13,085 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:13,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T20:47:13,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:13,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481293295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:13,296 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:13,296 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:13,296 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:13,484 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=164, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/95075228b1e346cca3297a0befa451a0 2024-11-24T20:47:13,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/f963b7669c4c438ea8504154537fdf4a is 50, key is test_row_0/B:col10/1732481230667/Put/seqid=0 2024-11-24T20:47:13,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741979_1155 (size=12151) 2024-11-24T20:47:13,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:13,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481293605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:13,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:13,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:13,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:13,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T20:47:13,895 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/f963b7669c4c438ea8504154537fdf4a 2024-11-24T20:47:13,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/45557526582347c69e4ad9c4c2177e52 is 50, key is test_row_0/C:col10/1732481230667/Put/seqid=0 2024-11-24T20:47:13,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741980_1156 (size=12151) 2024-11-24T20:47:13,915 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/45557526582347c69e4ad9c4c2177e52 2024-11-24T20:47:13,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/95075228b1e346cca3297a0befa451a0 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/95075228b1e346cca3297a0befa451a0 2024-11-24T20:47:13,924 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/95075228b1e346cca3297a0befa451a0, entries=150, sequenceid=164, filesize=30.4 K 2024-11-24T20:47:13,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/f963b7669c4c438ea8504154537fdf4a as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/f963b7669c4c438ea8504154537fdf4a 2024-11-24T20:47:13,931 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/f963b7669c4c438ea8504154537fdf4a, entries=150, sequenceid=164, filesize=11.9 K 2024-11-24T20:47:13,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/45557526582347c69e4ad9c4c2177e52 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/45557526582347c69e4ad9c4c2177e52 2024-11-24T20:47:13,939 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/45557526582347c69e4ad9c4c2177e52, entries=150, sequenceid=164, filesize=11.9 K 2024-11-24T20:47:13,940 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 170c4f62dfd32d7df8a14752b4c9e889 in 1296ms, sequenceid=164, compaction requested=true 2024-11-24T20:47:13,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 170c4f62dfd32d7df8a14752b4c9e889: 2024-11-24T20:47:13,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:13,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-24T20:47:13,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-24T20:47:13,945 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-24T20:47:13,945 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4530 sec 2024-11-24T20:47:13,947 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.4590 sec 2024-11-24T20:47:14,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:14,121 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:47:14,121 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 170c4f62dfd32d7df8a14752b4c9e889 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-24T20:47:14,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411247b262a928b214404a76c1edec80f4dfc_170c4f62dfd32d7df8a14752b4c9e889 is 50, key is test_row_0/A:col10/1732481234120/Put/seqid=0 2024-11-24T20:47:14,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741981_1157 (size=14794) 2024-11-24T20:47:14,162 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:14,166 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411247b262a928b214404a76c1edec80f4dfc_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247b262a928b214404a76c1edec80f4dfc_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:14,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:14,167 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/3af31d214c5b48219c2f2721098f5cea, store: [table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:14,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481294160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:14,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:14,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/3af31d214c5b48219c2f2721098f5cea is 175, key is test_row_0/A:col10/1732481234120/Put/seqid=0 2024-11-24T20:47:14,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:14,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:14,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741982_1158 (size=39749) 2024-11-24T20:47:14,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:14,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481294275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:14,283 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:14,283 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:14,283 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:14,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481294495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:14,587 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=190, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/3af31d214c5b48219c2f2721098f5cea 2024-11-24T20:47:14,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/b40a9ed502c7403e9ca62f12f2530af9 is 50, key is test_row_0/B:col10/1732481234120/Put/seqid=0 2024-11-24T20:47:14,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741983_1159 (size=12151) 2024-11-24T20:47:14,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T20:47:14,635 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:47:14,636 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:47:14,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-24T20:47:14,638 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:47:14,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T20:47:14,638 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:47:14,638 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:47:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T20:47:14,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T20:47:14,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:14,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:14,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:14,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:14,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:14,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:14,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:14,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481294805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:14,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:14,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:14,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:14,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T20:47:14,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:14,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:14,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:14,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:14,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:14,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:14,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T20:47:14,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49508 deadline: 1732481294975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:14,983 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_1' in 170c4f62dfd32d7df8a14752b4c9e889 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:47:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:14,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:14,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49456 deadline: 1732481294985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:14,985 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_2' in 170c4f62dfd32d7df8a14752b4c9e889 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:47:14,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:14,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:14,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:14,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:15,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49466 deadline: 1732481294995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:15,000 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_1' in 170c4f62dfd32d7df8a14752b4c9e889 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:47:15,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:15,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:15,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:15,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/b40a9ed502c7403e9ca62f12f2530af9 2024-11-24T20:47:15,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/65fdbb3128f24e6a898307260e12abf5 is 50, key is test_row_0/C:col10/1732481234120/Put/seqid=0 2024-11-24T20:47:15,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741984_1160 (size=12151) 2024-11-24T20:47:15,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T20:47:15,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:15,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:15,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:15,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:15,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:15,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:15,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T20:47:15,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:15,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:15,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:15,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:15,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T20:47:15,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:15,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481295315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:15,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:15,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:15,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:15,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T20:47:15,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:15,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:15,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:15,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:15,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:15,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:15,414 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/65fdbb3128f24e6a898307260e12abf5 2024-11-24T20:47:15,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/3af31d214c5b48219c2f2721098f5cea as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/3af31d214c5b48219c2f2721098f5cea 2024-11-24T20:47:15,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/3af31d214c5b48219c2f2721098f5cea, entries=200, sequenceid=190, filesize=38.8 K 2024-11-24T20:47:15,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/b40a9ed502c7403e9ca62f12f2530af9 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/b40a9ed502c7403e9ca62f12f2530af9 2024-11-24T20:47:15,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/b40a9ed502c7403e9ca62f12f2530af9, entries=150, sequenceid=190, filesize=11.9 K 2024-11-24T20:47:15,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/65fdbb3128f24e6a898307260e12abf5 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/65fdbb3128f24e6a898307260e12abf5 2024-11-24T20:47:15,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/65fdbb3128f24e6a898307260e12abf5, entries=150, sequenceid=190, filesize=11.9 K 2024-11-24T20:47:15,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 170c4f62dfd32d7df8a14752b4c9e889 in 1316ms, sequenceid=190, compaction requested=true 2024-11-24T20:47:15,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 170c4f62dfd32d7df8a14752b4c9e889: 2024-11-24T20:47:15,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 170c4f62dfd32d7df8a14752b4c9e889:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:47:15,437 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-24T20:47:15,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:15,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 170c4f62dfd32d7df8a14752b4c9e889:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:47:15,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:15,437 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-24T20:47:15,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 170c4f62dfd32d7df8a14752b4c9e889:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:47:15,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:47:15,440 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 82251 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-24T20:47:15,440 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 225913 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-24T20:47:15,440 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 170c4f62dfd32d7df8a14752b4c9e889/A is initiating minor compaction (all files) 2024-11-24T20:47:15,440 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 170c4f62dfd32d7df8a14752b4c9e889/B is initiating minor compaction (all files) 2024-11-24T20:47:15,440 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 170c4f62dfd32d7df8a14752b4c9e889/B in TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:15,440 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 170c4f62dfd32d7df8a14752b4c9e889/A in TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:15,440 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/2eb733e6c6c34c879b22af1e097adcd3, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/b6d59cda61ca41ada27c76565e152153, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/f73e78ac1d2c49af9f52070b70967af0, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/3887e781204b46bc840bf0e93fb81799, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/73e52585e63d4cbd84093d2a9d5353be, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/f963b7669c4c438ea8504154537fdf4a, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/b40a9ed502c7403e9ca62f12f2530af9] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp, totalSize=80.3 K 2024-11-24T20:47:15,440 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/afea27a9461b4b238744cf245fed946f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/3fc0a8a45f9e48788e303a7b3f442d83, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/8fa838d4a7924c3caa656a64099e794f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/61505695631d4a12a89ce20a456f1c56, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/40b6f82ef8d94a9cb351e7f518d5b0ab, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/95075228b1e346cca3297a0befa451a0, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/3af31d214c5b48219c2f2721098f5cea] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp, totalSize=220.6 K 2024-11-24T20:47:15,440 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=9 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:15,440 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. files: [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/afea27a9461b4b238744cf245fed946f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/3fc0a8a45f9e48788e303a7b3f442d83, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/8fa838d4a7924c3caa656a64099e794f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/61505695631d4a12a89ce20a456f1c56, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/40b6f82ef8d94a9cb351e7f518d5b0ab, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/95075228b1e346cca3297a0befa451a0, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/3af31d214c5b48219c2f2721098f5cea] 2024-11-24T20:47:15,440 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 2eb733e6c6c34c879b22af1e097adcd3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732481224175 2024-11-24T20:47:15,440 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting afea27a9461b4b238744cf245fed946f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732481224175 2024-11-24T20:47:15,441 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting b6d59cda61ca41ada27c76565e152153, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732481225391 2024-11-24T20:47:15,441 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fc0a8a45f9e48788e303a7b3f442d83, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732481225366 2024-11-24T20:47:15,442 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting f73e78ac1d2c49af9f52070b70967af0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732481226208 2024-11-24T20:47:15,442 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fa838d4a7924c3caa656a64099e794f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732481226208 2024-11-24T20:47:15,442 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 3887e781204b46bc840bf0e93fb81799, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732481226778 2024-11-24T20:47:15,442 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61505695631d4a12a89ce20a456f1c56, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732481226778 2024-11-24T20:47:15,442 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40b6f82ef8d94a9cb351e7f518d5b0ab, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732481228451 2024-11-24T20:47:15,442 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 73e52585e63d4cbd84093d2a9d5353be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732481228451 2024-11-24T20:47:15,442 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting f963b7669c4c438ea8504154537fdf4a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732481230664 2024-11-24T20:47:15,442 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95075228b1e346cca3297a0befa451a0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732481230664 2024-11-24T20:47:15,443 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting b40a9ed502c7403e9ca62f12f2530af9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732481232907 2024-11-24T20:47:15,443 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3af31d214c5b48219c2f2721098f5cea, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732481232907 2024-11-24T20:47:15,457 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 170c4f62dfd32d7df8a14752b4c9e889#B#compaction#134 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:15,458 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/451f52bda39a4e27b3ed4946f295c301 is 50, key is test_row_0/B:col10/1732481234120/Put/seqid=0 2024-11-24T20:47:15,461 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:15,464 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411246fa0ee0638424cd886431f71dd312966_170c4f62dfd32d7df8a14752b4c9e889 store=[table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:15,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741985_1161 (size=12527) 2024-11-24T20:47:15,468 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411246fa0ee0638424cd886431f71dd312966_170c4f62dfd32d7df8a14752b4c9e889, store=[table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:15,468 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411246fa0ee0638424cd886431f71dd312966_170c4f62dfd32d7df8a14752b4c9e889 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:15,477 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/451f52bda39a4e27b3ed4946f295c301 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/451f52bda39a4e27b3ed4946f295c301 2024-11-24T20:47:15,484 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 170c4f62dfd32d7df8a14752b4c9e889/B of 170c4f62dfd32d7df8a14752b4c9e889 into 451f52bda39a4e27b3ed4946f295c301(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:15,484 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 170c4f62dfd32d7df8a14752b4c9e889: 2024-11-24T20:47:15,484 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., storeName=170c4f62dfd32d7df8a14752b4c9e889/B, priority=9, startTime=1732481235437; duration=0sec 2024-11-24T20:47:15,484 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:47:15,484 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 170c4f62dfd32d7df8a14752b4c9e889:B 2024-11-24T20:47:15,484 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-24T20:47:15,487 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 82251 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-24T20:47:15,487 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 170c4f62dfd32d7df8a14752b4c9e889/C is initiating minor compaction (all files) 2024-11-24T20:47:15,488 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 170c4f62dfd32d7df8a14752b4c9e889/C in TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:15,488 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/1fb65818a076478e80f53807e56d460c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/e0b21b2505694fa7b1fa221256087aab, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/91dcfd6ad3d44f5b9c522bf7b55eef25, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/c79e80d624c846c98200f3eb980574fb, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/bf7eec729d494bc2b64429575d46abf3, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/45557526582347c69e4ad9c4c2177e52, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/65fdbb3128f24e6a898307260e12abf5] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp, totalSize=80.3 K 2024-11-24T20:47:15,488 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fb65818a076478e80f53807e56d460c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732481224175 2024-11-24T20:47:15,489 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting e0b21b2505694fa7b1fa221256087aab, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732481225391 2024-11-24T20:47:15,490 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 91dcfd6ad3d44f5b9c522bf7b55eef25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732481226208 2024-11-24T20:47:15,490 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting c79e80d624c846c98200f3eb980574fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732481226778 2024-11-24T20:47:15,490 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting bf7eec729d494bc2b64429575d46abf3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732481228451 2024-11-24T20:47:15,491 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 45557526582347c69e4ad9c4c2177e52, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732481230664 2024-11-24T20:47:15,491 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 65fdbb3128f24e6a898307260e12abf5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732481232907 2024-11-24T20:47:15,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741986_1162 (size=4469) 2024-11-24T20:47:15,515 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 170c4f62dfd32d7df8a14752b4c9e889#C#compaction#136 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:15,516 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/0745b8952e044fa08a65387b09829533 is 50, key is test_row_0/C:col10/1732481234120/Put/seqid=0 2024-11-24T20:47:15,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741987_1163 (size=12527) 2024-11-24T20:47:15,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T20:47:15,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:15,564 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 170c4f62dfd32d7df8a14752b4c9e889 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-24T20:47:15,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124a72a39f2473b4e899834fe074f53b52c_170c4f62dfd32d7df8a14752b4c9e889 is 50, key is test_row_0/A:col10/1732481234152/Put/seqid=0 2024-11-24T20:47:15,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741988_1164 (size=12304) 2024-11-24T20:47:15,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:15,590 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124a72a39f2473b4e899834fe074f53b52c_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124a72a39f2473b4e899834fe074f53b52c_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:15,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/d2371650466f41849f79f93b6ac5b632, store: [table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:15,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/d2371650466f41849f79f93b6ac5b632 is 175, key is test_row_0/A:col10/1732481234152/Put/seqid=0 2024-11-24T20:47:15,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741989_1165 (size=31105) 2024-11-24T20:47:15,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T20:47:15,906 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 170c4f62dfd32d7df8a14752b4c9e889#A#compaction#135 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:15,906 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/51c07530f1074930ac8e22a79a82d4b2 is 175, key is test_row_0/A:col10/1732481234120/Put/seqid=0 2024-11-24T20:47:15,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741990_1166 (size=31481) 2024-11-24T20:47:15,938 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/0745b8952e044fa08a65387b09829533 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/0745b8952e044fa08a65387b09829533 2024-11-24T20:47:15,944 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 170c4f62dfd32d7df8a14752b4c9e889/C of 170c4f62dfd32d7df8a14752b4c9e889 into 0745b8952e044fa08a65387b09829533(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:15,944 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 170c4f62dfd32d7df8a14752b4c9e889: 2024-11-24T20:47:15,944 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., storeName=170c4f62dfd32d7df8a14752b4c9e889/C, priority=9, startTime=1732481235437; duration=0sec 2024-11-24T20:47:15,944 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:15,944 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 170c4f62dfd32d7df8a14752b4c9e889:C 2024-11-24T20:47:16,009 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=201, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/d2371650466f41849f79f93b6ac5b632 2024-11-24T20:47:16,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/7420b22aa10347d3ba66cefcda44cadf is 50, key is test_row_0/B:col10/1732481234152/Put/seqid=0 2024-11-24T20:47:16,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741991_1167 (size=12151) 2024-11-24T20:47:16,323 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/51c07530f1074930ac8e22a79a82d4b2 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/51c07530f1074930ac8e22a79a82d4b2 2024-11-24T20:47:16,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:16,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:16,335 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 170c4f62dfd32d7df8a14752b4c9e889/A of 170c4f62dfd32d7df8a14752b4c9e889 into 51c07530f1074930ac8e22a79a82d4b2(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:16,335 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 170c4f62dfd32d7df8a14752b4c9e889: 2024-11-24T20:47:16,335 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., storeName=170c4f62dfd32d7df8a14752b4c9e889/A, priority=9, startTime=1732481235437; duration=0sec 2024-11-24T20:47:16,335 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:16,336 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 170c4f62dfd32d7df8a14752b4c9e889:A 2024-11-24T20:47:16,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:16,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481296421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:16,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:16,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:16,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:16,426 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/7420b22aa10347d3ba66cefcda44cadf 2024-11-24T20:47:16,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/16641a834e9c4f448a6cb3e42f602bf6 is 50, key is test_row_0/C:col10/1732481234152/Put/seqid=0 2024-11-24T20:47:16,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741992_1168 (size=12151) 2024-11-24T20:47:16,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:16,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481296535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:16,536 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:16,536 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:16,536 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:16,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:16,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481296745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:16,747 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:16,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:16,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:16,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T20:47:16,840 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/16641a834e9c4f448a6cb3e42f602bf6 2024-11-24T20:47:16,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/d2371650466f41849f79f93b6ac5b632 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/d2371650466f41849f79f93b6ac5b632 2024-11-24T20:47:16,848 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/d2371650466f41849f79f93b6ac5b632, entries=150, sequenceid=201, filesize=30.4 K 2024-11-24T20:47:16,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/7420b22aa10347d3ba66cefcda44cadf as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/7420b22aa10347d3ba66cefcda44cadf 2024-11-24T20:47:16,854 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/7420b22aa10347d3ba66cefcda44cadf, entries=150, sequenceid=201, filesize=11.9 K 2024-11-24T20:47:16,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/16641a834e9c4f448a6cb3e42f602bf6 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/16641a834e9c4f448a6cb3e42f602bf6 2024-11-24T20:47:16,861 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/16641a834e9c4f448a6cb3e42f602bf6, entries=150, sequenceid=201, filesize=11.9 K 2024-11-24T20:47:16,862 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 170c4f62dfd32d7df8a14752b4c9e889 in 1298ms, sequenceid=201, compaction requested=false 2024-11-24T20:47:16,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 170c4f62dfd32d7df8a14752b4c9e889: 2024-11-24T20:47:16,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:16,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-24T20:47:16,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-24T20:47:16,865 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-24T20:47:16,865 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2250 sec 2024-11-24T20:47:16,867 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.2300 sec 2024-11-24T20:47:17,058 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:47:17,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 170c4f62dfd32d7df8a14752b4c9e889 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-24T20:47:17,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:17,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124573976522303420fb449acf993c3b6b8_170c4f62dfd32d7df8a14752b4c9e889 is 50, key is test_row_0/A:col10/1732481236415/Put/seqid=0 2024-11-24T20:47:17,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741993_1169 (size=14794) 2024-11-24T20:47:17,076 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:17,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:17,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481297079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:17,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:17,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:17,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:17,084 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124573976522303420fb449acf993c3b6b8_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124573976522303420fb449acf993c3b6b8_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:17,087 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/319730d0ea7b4110bf7b53d0bb380c7a, store: [table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:17,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/319730d0ea7b4110bf7b53d0bb380c7a is 175, key is test_row_0/A:col10/1732481236415/Put/seqid=0 2024-11-24T20:47:17,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741994_1170 (size=39749) 2024-11-24T20:47:17,102 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=230, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/319730d0ea7b4110bf7b53d0bb380c7a 2024-11-24T20:47:17,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/b79a93c0c7ac4b859b47291649141c00 is 50, key is test_row_0/B:col10/1732481236415/Put/seqid=0 2024-11-24T20:47:17,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741995_1171 (size=12151) 2024-11-24T20:47:17,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/b79a93c0c7ac4b859b47291649141c00 2024-11-24T20:47:17,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/aa57d140fa7f442aa458d9b3f702b11b is 50, key is test_row_0/C:col10/1732481236415/Put/seqid=0 2024-11-24T20:47:17,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741996_1172 (size=12151) 2024-11-24T20:47:17,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:17,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481297185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:17,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:17,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:17,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:17,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:17,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481297395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:17,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:17,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:17,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:17,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/aa57d140fa7f442aa458d9b3f702b11b 2024-11-24T20:47:17,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/319730d0ea7b4110bf7b53d0bb380c7a as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/319730d0ea7b4110bf7b53d0bb380c7a 2024-11-24T20:47:17,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/319730d0ea7b4110bf7b53d0bb380c7a, entries=200, sequenceid=230, filesize=38.8 K 2024-11-24T20:47:17,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/b79a93c0c7ac4b859b47291649141c00 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/b79a93c0c7ac4b859b47291649141c00 2024-11-24T20:47:17,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/b79a93c0c7ac4b859b47291649141c00, entries=150, sequenceid=230, filesize=11.9 K 2024-11-24T20:47:17,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/aa57d140fa7f442aa458d9b3f702b11b as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/aa57d140fa7f442aa458d9b3f702b11b 2024-11-24T20:47:17,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/aa57d140fa7f442aa458d9b3f702b11b, entries=150, sequenceid=230, filesize=11.9 K 2024-11-24T20:47:17,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 170c4f62dfd32d7df8a14752b4c9e889 in 521ms, sequenceid=230, compaction requested=true 2024-11-24T20:47:17,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 170c4f62dfd32d7df8a14752b4c9e889: 2024-11-24T20:47:17,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 170c4f62dfd32d7df8a14752b4c9e889:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:47:17,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:17,579 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:47:17,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 170c4f62dfd32d7df8a14752b4c9e889:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:47:17,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:17,579 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:47:17,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 170c4f62dfd32d7df8a14752b4c9e889:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:47:17,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:47:17,580 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:47:17,580 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:47:17,580 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 170c4f62dfd32d7df8a14752b4c9e889/B is initiating minor compaction (all files) 2024-11-24T20:47:17,580 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 170c4f62dfd32d7df8a14752b4c9e889/A is initiating minor compaction (all files) 2024-11-24T20:47:17,580 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 170c4f62dfd32d7df8a14752b4c9e889/B in TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:17,580 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 170c4f62dfd32d7df8a14752b4c9e889/A in TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:17,580 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/51c07530f1074930ac8e22a79a82d4b2, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/d2371650466f41849f79f93b6ac5b632, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/319730d0ea7b4110bf7b53d0bb380c7a] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp, totalSize=99.9 K 2024-11-24T20:47:17,580 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/451f52bda39a4e27b3ed4946f295c301, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/7420b22aa10347d3ba66cefcda44cadf, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/b79a93c0c7ac4b859b47291649141c00] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp, totalSize=36.0 K 2024-11-24T20:47:17,580 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:17,580 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. files: [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/51c07530f1074930ac8e22a79a82d4b2, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/d2371650466f41849f79f93b6ac5b632, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/319730d0ea7b4110bf7b53d0bb380c7a] 2024-11-24T20:47:17,581 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51c07530f1074930ac8e22a79a82d4b2, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732481232907 2024-11-24T20:47:17,581 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 451f52bda39a4e27b3ed4946f295c301, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732481232907 2024-11-24T20:47:17,581 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2371650466f41849f79f93b6ac5b632, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732481234135 2024-11-24T20:47:17,581 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 7420b22aa10347d3ba66cefcda44cadf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732481234135 2024-11-24T20:47:17,581 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 319730d0ea7b4110bf7b53d0bb380c7a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732481236401 2024-11-24T20:47:17,581 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting b79a93c0c7ac4b859b47291649141c00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732481236401 2024-11-24T20:47:17,587 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:17,590 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 170c4f62dfd32d7df8a14752b4c9e889#B#compaction#144 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:17,590 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/a73927c749e8492c8211e2abb25e9ed5 is 50, key is test_row_0/B:col10/1732481236415/Put/seqid=0 2024-11-24T20:47:17,591 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411240d69cdd8ec4247a2ba0e5d4b3d6275c5_170c4f62dfd32d7df8a14752b4c9e889 store=[table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:17,593 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411240d69cdd8ec4247a2ba0e5d4b3d6275c5_170c4f62dfd32d7df8a14752b4c9e889, store=[table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:17,593 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240d69cdd8ec4247a2ba0e5d4b3d6275c5_170c4f62dfd32d7df8a14752b4c9e889 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:17,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741998_1174 (size=4469) 2024-11-24T20:47:17,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741997_1173 (size=12629) 2024-11-24T20:47:17,709 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:47:17,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:17,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 170c4f62dfd32d7df8a14752b4c9e889 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T20:47:17,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124295d653563f94c2a91e9523c64bbfa81_170c4f62dfd32d7df8a14752b4c9e889 is 50, key is test_row_0/A:col10/1732481237708/Put/seqid=0 2024-11-24T20:47:17,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741999_1175 (size=17284) 2024-11-24T20:47:17,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481297802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:17,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481297915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:17,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:17,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:17,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:17,999 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 170c4f62dfd32d7df8a14752b4c9e889#A#compaction#143 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:17,999 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/c9c22cbe8b6d4563a76346080005864c is 175, key is test_row_0/A:col10/1732481236415/Put/seqid=0 2024-11-24T20:47:18,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742000_1176 (size=31583) 2024-11-24T20:47:18,009 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/a73927c749e8492c8211e2abb25e9ed5 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/a73927c749e8492c8211e2abb25e9ed5 2024-11-24T20:47:18,014 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 170c4f62dfd32d7df8a14752b4c9e889/B of 170c4f62dfd32d7df8a14752b4c9e889 into a73927c749e8492c8211e2abb25e9ed5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:18,014 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 170c4f62dfd32d7df8a14752b4c9e889: 2024-11-24T20:47:18,014 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., storeName=170c4f62dfd32d7df8a14752b4c9e889/B, priority=13, startTime=1732481237579; duration=0sec 2024-11-24T20:47:18,014 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:47:18,014 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 170c4f62dfd32d7df8a14752b4c9e889:B 2024-11-24T20:47:18,014 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/c9c22cbe8b6d4563a76346080005864c as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/c9c22cbe8b6d4563a76346080005864c 2024-11-24T20:47:18,014 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:47:18,016 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:47:18,016 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 170c4f62dfd32d7df8a14752b4c9e889/C is initiating minor compaction (all files) 2024-11-24T20:47:18,016 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 170c4f62dfd32d7df8a14752b4c9e889/C in TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:18,016 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/0745b8952e044fa08a65387b09829533, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/16641a834e9c4f448a6cb3e42f602bf6, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/aa57d140fa7f442aa458d9b3f702b11b] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp, totalSize=36.0 K 2024-11-24T20:47:18,017 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 0745b8952e044fa08a65387b09829533, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732481232907 2024-11-24T20:47:18,017 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 16641a834e9c4f448a6cb3e42f602bf6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732481234135 2024-11-24T20:47:18,018 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting aa57d140fa7f442aa458d9b3f702b11b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732481236401 2024-11-24T20:47:18,020 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 170c4f62dfd32d7df8a14752b4c9e889/A of 170c4f62dfd32d7df8a14752b4c9e889 into c9c22cbe8b6d4563a76346080005864c(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:18,021 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 170c4f62dfd32d7df8a14752b4c9e889: 2024-11-24T20:47:18,021 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., storeName=170c4f62dfd32d7df8a14752b4c9e889/A, priority=13, startTime=1732481237579; duration=0sec 2024-11-24T20:47:18,021 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:18,021 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 170c4f62dfd32d7df8a14752b4c9e889:A 2024-11-24T20:47:18,026 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 170c4f62dfd32d7df8a14752b4c9e889#C#compaction#146 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:18,026 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/3a4ed5497593499aa7cdae367cae7985 is 50, key is test_row_0/C:col10/1732481236415/Put/seqid=0 2024-11-24T20:47:18,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742001_1177 (size=12629) 2024-11-24T20:47:18,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:18,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481298125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:18,128 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:18,128 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:18,128 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:18,131 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:18,136 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124295d653563f94c2a91e9523c64bbfa81_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124295d653563f94c2a91e9523c64bbfa81_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:18,137 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/cb287ca18f294aa093b31b9d750b8e78, store: [table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:18,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/cb287ca18f294aa093b31b9d750b8e78 is 175, key is test_row_0/A:col10/1732481237708/Put/seqid=0 2024-11-24T20:47:18,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742002_1178 (size=48389) 2024-11-24T20:47:18,435 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/3a4ed5497593499aa7cdae367cae7985 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/3a4ed5497593499aa7cdae367cae7985 2024-11-24T20:47:18,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:18,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481298435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:18,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:18,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:18,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:18,442 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 170c4f62dfd32d7df8a14752b4c9e889/C of 170c4f62dfd32d7df8a14752b4c9e889 into 3a4ed5497593499aa7cdae367cae7985(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:18,442 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 170c4f62dfd32d7df8a14752b4c9e889: 2024-11-24T20:47:18,442 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., storeName=170c4f62dfd32d7df8a14752b4c9e889/C, priority=13, startTime=1732481237579; duration=0sec 2024-11-24T20:47:18,442 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:18,442 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 170c4f62dfd32d7df8a14752b4c9e889:C 2024-11-24T20:47:18,546 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=241, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/cb287ca18f294aa093b31b9d750b8e78 2024-11-24T20:47:18,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/079698bfafa444b3b21a9937454b0133 is 50, key is test_row_0/B:col10/1732481237708/Put/seqid=0 2024-11-24T20:47:18,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742003_1179 (size=12151) 2024-11-24T20:47:18,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/079698bfafa444b3b21a9937454b0133 2024-11-24T20:47:18,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/d5ce1b45ecae41759717c006a0a274c1 is 50, key is test_row_0/C:col10/1732481237708/Put/seqid=0 2024-11-24T20:47:18,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742004_1180 (size=12151) 2024-11-24T20:47:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T20:47:18,795 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:47:18,795 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:47:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-24T20:47:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T20:47:18,797 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:47:18,798 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:47:18,798 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:47:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T20:47:18,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481298945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:18,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-24T20:47:18,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:18,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:18,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:18,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:18,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:18,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/d5ce1b45ecae41759717c006a0a274c1 2024-11-24T20:47:18,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/cb287ca18f294aa093b31b9d750b8e78 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/cb287ca18f294aa093b31b9d750b8e78 2024-11-24T20:47:19,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/cb287ca18f294aa093b31b9d750b8e78, entries=250, sequenceid=241, filesize=47.3 K 2024-11-24T20:47:19,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/079698bfafa444b3b21a9937454b0133 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/079698bfafa444b3b21a9937454b0133 2024-11-24T20:47:19,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/079698bfafa444b3b21a9937454b0133, entries=150, sequenceid=241, filesize=11.9 K 2024-11-24T20:47:19,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/d5ce1b45ecae41759717c006a0a274c1 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/d5ce1b45ecae41759717c006a0a274c1 2024-11-24T20:47:19,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/d5ce1b45ecae41759717c006a0a274c1, entries=150, sequenceid=241, filesize=11.9 K 2024-11-24T20:47:19,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 170c4f62dfd32d7df8a14752b4c9e889 in 1302ms, sequenceid=241, compaction requested=false 2024-11-24T20:47:19,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 170c4f62dfd32d7df8a14752b4c9e889: 2024-11-24T20:47:19,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-24T20:47:19,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:19,104 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 170c4f62dfd32d7df8a14752b4c9e889 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-24T20:47:19,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241e84e71a4136499d972099e0eb6491dd_170c4f62dfd32d7df8a14752b4c9e889 is 50, key is test_row_0/A:col10/1732481237781/Put/seqid=0 2024-11-24T20:47:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T20:47:19,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742005_1181 (size=12454) 2024-11-24T20:47:19,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:19,129 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241e84e71a4136499d972099e0eb6491dd_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241e84e71a4136499d972099e0eb6491dd_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:19,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/4d1ed7e972e3472082be7bc7553796a1, store: [table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:19,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/4d1ed7e972e3472082be7bc7553796a1 is 175, key is test_row_0/A:col10/1732481237781/Put/seqid=0 2024-11-24T20:47:19,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742006_1182 (size=31255) 2024-11-24T20:47:19,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T20:47:19,538 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=269, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/4d1ed7e972e3472082be7bc7553796a1 2024-11-24T20:47:19,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/2bc71dd1b35e4c109749a02ca89f82b9 is 50, key is test_row_0/B:col10/1732481237781/Put/seqid=0 2024-11-24T20:47:19,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742007_1183 (size=12301) 2024-11-24T20:47:19,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T20:47:19,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:19,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:19,959 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/2bc71dd1b35e4c109749a02ca89f82b9 2024-11-24T20:47:19,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/e4240b4fb93c47f29992ede18ff24ac3 is 50, key is test_row_0/C:col10/1732481237781/Put/seqid=0 2024-11-24T20:47:19,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742008_1184 (size=12301) 2024-11-24T20:47:19,976 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/e4240b4fb93c47f29992ede18ff24ac3 2024-11-24T20:47:19,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:19,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481299979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:19,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:19,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:19,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:19,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/4d1ed7e972e3472082be7bc7553796a1 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/4d1ed7e972e3472082be7bc7553796a1 2024-11-24T20:47:19,986 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/4d1ed7e972e3472082be7bc7553796a1, entries=150, sequenceid=269, filesize=30.5 K 2024-11-24T20:47:19,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/2bc71dd1b35e4c109749a02ca89f82b9 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/2bc71dd1b35e4c109749a02ca89f82b9 2024-11-24T20:47:19,993 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/2bc71dd1b35e4c109749a02ca89f82b9, entries=150, sequenceid=269, filesize=12.0 K 2024-11-24T20:47:19,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/e4240b4fb93c47f29992ede18ff24ac3 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/e4240b4fb93c47f29992ede18ff24ac3 2024-11-24T20:47:20,003 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/e4240b4fb93c47f29992ede18ff24ac3, entries=150, sequenceid=269, filesize=12.0 K 2024-11-24T20:47:20,004 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 170c4f62dfd32d7df8a14752b4c9e889 in 900ms, sequenceid=269, compaction requested=true 2024-11-24T20:47:20,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 170c4f62dfd32d7df8a14752b4c9e889: 2024-11-24T20:47:20,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:20,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-24T20:47:20,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-24T20:47:20,007 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-24T20:47:20,007 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2070 sec 2024-11-24T20:47:20,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.2120 sec 2024-11-24T20:47:20,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:20,087 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:47:20,087 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 170c4f62dfd32d7df8a14752b4c9e889 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-24T20:47:20,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124bdfb07be1e344e7c9a1c0c5b76043ae3_170c4f62dfd32d7df8a14752b4c9e889 is 50, key is test_row_0/A:col10/1732481239965/Put/seqid=0 2024-11-24T20:47:20,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742009_1185 (size=14994) 2024-11-24T20:47:20,105 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:20,109 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124bdfb07be1e344e7c9a1c0c5b76043ae3_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bdfb07be1e344e7c9a1c0c5b76043ae3_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:20,110 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/eee172fb67864c959a6735024f7917fa, store: [table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:20,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/eee172fb67864c959a6735024f7917fa is 175, key is test_row_0/A:col10/1732481239965/Put/seqid=0 2024-11-24T20:47:20,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742010_1186 (size=39949) 2024-11-24T20:47:20,115 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/eee172fb67864c959a6735024f7917fa 2024-11-24T20:47:20,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/27b6f33b38324898946a329af1c49ea1 is 50, key is test_row_0/B:col10/1732481239965/Put/seqid=0 2024-11-24T20:47:20,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742011_1187 (size=12301) 2024-11-24T20:47:20,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/27b6f33b38324898946a329af1c49ea1 2024-11-24T20:47:20,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/ba559abb183248758cac39f3e25ab579 is 50, key is test_row_0/C:col10/1732481239965/Put/seqid=0 2024-11-24T20:47:20,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742012_1188 (size=12301) 2024-11-24T20:47:20,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/ba559abb183248758cac39f3e25ab579 2024-11-24T20:47:20,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/eee172fb67864c959a6735024f7917fa as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/eee172fb67864c959a6735024f7917fa 2024-11-24T20:47:20,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/eee172fb67864c959a6735024f7917fa, entries=200, sequenceid=281, filesize=39.0 K 2024-11-24T20:47:20,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/27b6f33b38324898946a329af1c49ea1 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/27b6f33b38324898946a329af1c49ea1 2024-11-24T20:47:20,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/27b6f33b38324898946a329af1c49ea1, entries=150, sequenceid=281, filesize=12.0 K 2024-11-24T20:47:20,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/ba559abb183248758cac39f3e25ab579 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/ba559abb183248758cac39f3e25ab579 2024-11-24T20:47:20,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/ba559abb183248758cac39f3e25ab579, entries=150, sequenceid=281, filesize=12.0 K 2024-11-24T20:47:20,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=120.76 KB/123660 for 170c4f62dfd32d7df8a14752b4c9e889 in 91ms, sequenceid=281, compaction requested=true 2024-11-24T20:47:20,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 170c4f62dfd32d7df8a14752b4c9e889: 2024-11-24T20:47:20,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 170c4f62dfd32d7df8a14752b4c9e889:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:47:20,178 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:47:20,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:20,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 170c4f62dfd32d7df8a14752b4c9e889:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:47:20,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:20,178 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:47:20,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 170c4f62dfd32d7df8a14752b4c9e889:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:47:20,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:47:20,179 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151176 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:47:20,179 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49382 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:47:20,179 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 170c4f62dfd32d7df8a14752b4c9e889/B is initiating minor compaction (all files) 2024-11-24T20:47:20,179 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 170c4f62dfd32d7df8a14752b4c9e889/A is initiating minor compaction (all files) 2024-11-24T20:47:20,179 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 170c4f62dfd32d7df8a14752b4c9e889/A in TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:20,179 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 170c4f62dfd32d7df8a14752b4c9e889/B in TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:20,179 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/a73927c749e8492c8211e2abb25e9ed5, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/079698bfafa444b3b21a9937454b0133, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/2bc71dd1b35e4c109749a02ca89f82b9, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/27b6f33b38324898946a329af1c49ea1] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp, totalSize=48.2 K 2024-11-24T20:47:20,179 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/c9c22cbe8b6d4563a76346080005864c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/cb287ca18f294aa093b31b9d750b8e78, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/4d1ed7e972e3472082be7bc7553796a1, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/eee172fb67864c959a6735024f7917fa] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp, totalSize=147.6 K 2024-11-24T20:47:20,180 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:20,180 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. files: [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/c9c22cbe8b6d4563a76346080005864c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/cb287ca18f294aa093b31b9d750b8e78, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/4d1ed7e972e3472082be7bc7553796a1, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/eee172fb67864c959a6735024f7917fa] 2024-11-24T20:47:20,180 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting a73927c749e8492c8211e2abb25e9ed5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732481236401 2024-11-24T20:47:20,180 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9c22cbe8b6d4563a76346080005864c, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732481236401 2024-11-24T20:47:20,180 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb287ca18f294aa093b31b9d750b8e78, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732481237066 2024-11-24T20:47:20,180 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 079698bfafa444b3b21a9937454b0133, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732481237066 2024-11-24T20:47:20,181 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d1ed7e972e3472082be7bc7553796a1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732481237781 2024-11-24T20:47:20,181 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bc71dd1b35e4c109749a02ca89f82b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732481237781 2024-11-24T20:47:20,181 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting eee172fb67864c959a6735024f7917fa, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732481239965 2024-11-24T20:47:20,181 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 27b6f33b38324898946a329af1c49ea1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732481239965 2024-11-24T20:47:20,182 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:47:20,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 170c4f62dfd32d7df8a14752b4c9e889 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-24T20:47:20,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:20,208 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 170c4f62dfd32d7df8a14752b4c9e889#B#compaction#155 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:20,209 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/791ea83d3f3042f89cb2643183644661 is 50, key is test_row_0/B:col10/1732481239965/Put/seqid=0 2024-11-24T20:47:20,210 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:20,214 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124dee546df041647079dac39cbfcfb1a33_170c4f62dfd32d7df8a14752b4c9e889 store=[table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:20,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124bf90a90a1a8145b1a2aca9ba44b3a5fa_170c4f62dfd32d7df8a14752b4c9e889 is 50, key is test_row_0/A:col10/1732481240171/Put/seqid=0 2024-11-24T20:47:20,216 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124dee546df041647079dac39cbfcfb1a33_170c4f62dfd32d7df8a14752b4c9e889, store=[table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:20,216 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124dee546df041647079dac39cbfcfb1a33_170c4f62dfd32d7df8a14752b4c9e889 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:20,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742014_1190 (size=17534) 2024-11-24T20:47:20,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742015_1191 (size=4469) 2024-11-24T20:47:20,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742013_1189 (size=12915) 2024-11-24T20:47:20,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:20,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481300246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:20,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:20,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:20,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:20,252 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/791ea83d3f3042f89cb2643183644661 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/791ea83d3f3042f89cb2643183644661 2024-11-24T20:47:20,258 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 170c4f62dfd32d7df8a14752b4c9e889/B of 170c4f62dfd32d7df8a14752b4c9e889 into 791ea83d3f3042f89cb2643183644661(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:20,258 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 170c4f62dfd32d7df8a14752b4c9e889: 2024-11-24T20:47:20,258 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., storeName=170c4f62dfd32d7df8a14752b4c9e889/B, priority=12, startTime=1732481240178; duration=0sec 2024-11-24T20:47:20,259 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:47:20,259 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 170c4f62dfd32d7df8a14752b4c9e889:B 2024-11-24T20:47:20,259 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:47:20,260 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49382 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:47:20,260 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 170c4f62dfd32d7df8a14752b4c9e889/C is initiating minor compaction (all files) 2024-11-24T20:47:20,260 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 170c4f62dfd32d7df8a14752b4c9e889/C in TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:20,260 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/3a4ed5497593499aa7cdae367cae7985, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/d5ce1b45ecae41759717c006a0a274c1, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/e4240b4fb93c47f29992ede18ff24ac3, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/ba559abb183248758cac39f3e25ab579] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp, totalSize=48.2 K 2024-11-24T20:47:20,262 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a4ed5497593499aa7cdae367cae7985, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732481236401 2024-11-24T20:47:20,262 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting d5ce1b45ecae41759717c006a0a274c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732481237066 2024-11-24T20:47:20,263 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting e4240b4fb93c47f29992ede18ff24ac3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732481237781 2024-11-24T20:47:20,264 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting ba559abb183248758cac39f3e25ab579, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732481239965 2024-11-24T20:47:20,276 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 170c4f62dfd32d7df8a14752b4c9e889#C#compaction#158 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:20,277 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/5e6b7bf41b51407593250e6b14831570 is 50, key is test_row_0/C:col10/1732481239965/Put/seqid=0 2024-11-24T20:47:20,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742016_1192 (size=12915) 2024-11-24T20:47:20,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481300355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:20,357 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:20,357 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:20,357 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:20,443 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T20:47:20,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:20,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481300565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:20,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:20,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:20,569 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:20,638 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:20,642 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124bf90a90a1a8145b1a2aca9ba44b3a5fa_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bf90a90a1a8145b1a2aca9ba44b3a5fa_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:20,644 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/ca68dd45e76648bd88c79e1aec2c2e9f, store: [table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:20,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/ca68dd45e76648bd88c79e1aec2c2e9f is 175, key is test_row_0/A:col10/1732481240171/Put/seqid=0 2024-11-24T20:47:20,644 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 170c4f62dfd32d7df8a14752b4c9e889#A#compaction#156 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:20,645 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/7146399566fc4c848c71613d23ed142d is 175, key is test_row_0/A:col10/1732481239965/Put/seqid=0 2024-11-24T20:47:20,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742017_1193 (size=48639) 2024-11-24T20:47:20,650 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=303, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/ca68dd45e76648bd88c79e1aec2c2e9f 2024-11-24T20:47:20,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742018_1194 (size=31869) 2024-11-24T20:47:20,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/e1408808f19f4bd1b81c33365b272481 is 50, key is test_row_0/B:col10/1732481240171/Put/seqid=0 2024-11-24T20:47:20,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742019_1195 (size=12301) 2024-11-24T20:47:20,686 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/5e6b7bf41b51407593250e6b14831570 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/5e6b7bf41b51407593250e6b14831570 2024-11-24T20:47:20,692 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 170c4f62dfd32d7df8a14752b4c9e889/C of 170c4f62dfd32d7df8a14752b4c9e889 into 5e6b7bf41b51407593250e6b14831570(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:20,692 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 170c4f62dfd32d7df8a14752b4c9e889: 2024-11-24T20:47:20,692 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., storeName=170c4f62dfd32d7df8a14752b4c9e889/C, priority=12, startTime=1732481240178; duration=0sec 2024-11-24T20:47:20,692 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:20,692 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 170c4f62dfd32d7df8a14752b4c9e889:C 2024-11-24T20:47:20,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:20,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732481300855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:20,857 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_2' in 170c4f62dfd32d7df8a14752b4c9e889 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18272 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:47:20,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:20,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:20,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:20,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:20,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481300875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:20,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:20,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:20,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T20:47:20,945 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:47:20,945 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:47:20,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-24T20:47:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T20:47:20,947 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:47:20,948 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:47:20,948 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:47:21,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T20:47:21,067 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/7146399566fc4c848c71613d23ed142d as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/7146399566fc4c848c71613d23ed142d 2024-11-24T20:47:21,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/e1408808f19f4bd1b81c33365b272481 2024-11-24T20:47:21,072 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 170c4f62dfd32d7df8a14752b4c9e889/A of 170c4f62dfd32d7df8a14752b4c9e889 into 7146399566fc4c848c71613d23ed142d(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:21,072 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 170c4f62dfd32d7df8a14752b4c9e889: 2024-11-24T20:47:21,072 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., storeName=170c4f62dfd32d7df8a14752b4c9e889/A, priority=12, startTime=1732481240178; duration=0sec 2024-11-24T20:47:21,072 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:21,072 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 170c4f62dfd32d7df8a14752b4c9e889:A 2024-11-24T20:47:21,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/cd9a5a95d6c54b91bca61a6a04d51174 is 50, key is test_row_0/C:col10/1732481240171/Put/seqid=0 2024-11-24T20:47:21,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742020_1196 (size=12301) 2024-11-24T20:47:21,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-24T20:47:21,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:21,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:21,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:21,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:21,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:21,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:21,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-24T20:47:21,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:21,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:21,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:21,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:21,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:21,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:21,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T20:47:21,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:21,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732481301385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:21,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:21,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:21,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:47:21,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-24T20:47:21,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:21,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:21,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:21,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:21,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:21,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:21,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/cd9a5a95d6c54b91bca61a6a04d51174 2024-11-24T20:47:21,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/ca68dd45e76648bd88c79e1aec2c2e9f as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/ca68dd45e76648bd88c79e1aec2c2e9f 2024-11-24T20:47:21,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/ca68dd45e76648bd88c79e1aec2c2e9f, entries=250, sequenceid=303, filesize=47.5 K 2024-11-24T20:47:21,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/e1408808f19f4bd1b81c33365b272481 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/e1408808f19f4bd1b81c33365b272481 2024-11-24T20:47:21,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/e1408808f19f4bd1b81c33365b272481, entries=150, sequenceid=303, filesize=12.0 K 2024-11-24T20:47:21,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/cd9a5a95d6c54b91bca61a6a04d51174 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/cd9a5a95d6c54b91bca61a6a04d51174 2024-11-24T20:47:21,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/cd9a5a95d6c54b91bca61a6a04d51174, entries=150, sequenceid=303, filesize=12.0 K 2024-11-24T20:47:21,521 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 170c4f62dfd32d7df8a14752b4c9e889 in 1338ms, sequenceid=303, compaction requested=false 2024-11-24T20:47:21,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 170c4f62dfd32d7df8a14752b4c9e889: 2024-11-24T20:47:21,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-24T20:47:21,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:21,560 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 170c4f62dfd32d7df8a14752b4c9e889 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-24T20:47:21,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411242f4a1749d2bb43edb5feea82dac4d9d3_170c4f62dfd32d7df8a14752b4c9e889 is 50, key is test_row_0/A:col10/1732481240244/Put/seqid=0 2024-11-24T20:47:21,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742021_1197 (size=12454) 2024-11-24T20:47:21,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T20:47:21,846 INFO [Thread-607 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-607. 2024-11-24T20:47:21,846 DEBUG [Thread-607 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:47:21,846 DEBUG [Thread-607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:21,846 DEBUG [Thread-607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:21,846 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:47:21,847 INFO [Thread-611 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-611. 2024-11-24T20:47:21,847 DEBUG [Thread-611 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:47:21,847 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:21,848 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:21,848 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:47:21,848 INFO [Thread-605 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-605. 2024-11-24T20:47:21,848 DEBUG [Thread-605 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:47:21,848 DEBUG [Thread-605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:21,848 DEBUG [Thread-605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:21,848 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:47:21,851 INFO [Thread-609 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-609. 2024-11-24T20:47:21,851 DEBUG [Thread-609 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:47:21,851 DEBUG [Thread-609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:21,851 DEBUG [Thread-609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:21,851 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:47:21,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:21,981 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411242f4a1749d2bb43edb5feea82dac4d9d3_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242f4a1749d2bb43edb5feea82dac4d9d3_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:21,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/cbd1564ae0534910a164bbcf63477a48, store: [table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:21,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/cbd1564ae0534910a164bbcf63477a48 is 175, key is test_row_0/A:col10/1732481240244/Put/seqid=0 2024-11-24T20:47:21,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742022_1198 (size=31255) 2024-11-24T20:47:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T20:47:22,388 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=320, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/cbd1564ae0534910a164bbcf63477a48 2024-11-24T20:47:22,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/703522dfed194cbfb8c4d56984fdf0bb is 50, key is test_row_0/B:col10/1732481240244/Put/seqid=0 2024-11-24T20:47:22,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742023_1199 (size=12301) 2024-11-24T20:47:22,401 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/703522dfed194cbfb8c4d56984fdf0bb 2024-11-24T20:47:22,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. as already flushing 2024-11-24T20:47:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:22,406 INFO [Thread-602 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-602. 2024-11-24T20:47:22,406 DEBUG [Thread-602 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:47:22,406 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:22,406 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:22,406 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:47:22,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/550e44635f604dd781fedfdacadb5c2d is 50, key is test_row_0/C:col10/1732481240244/Put/seqid=0 2024-11-24T20:47:22,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742024_1200 (size=12301) 2024-11-24T20:47:22,815 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/550e44635f604dd781fedfdacadb5c2d 2024-11-24T20:47:22,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/cbd1564ae0534910a164bbcf63477a48 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/cbd1564ae0534910a164bbcf63477a48 2024-11-24T20:47:22,825 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/cbd1564ae0534910a164bbcf63477a48, entries=150, sequenceid=320, filesize=30.5 K 2024-11-24T20:47:22,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/703522dfed194cbfb8c4d56984fdf0bb as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/703522dfed194cbfb8c4d56984fdf0bb 2024-11-24T20:47:22,830 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/703522dfed194cbfb8c4d56984fdf0bb, entries=150, sequenceid=320, filesize=12.0 K 2024-11-24T20:47:22,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/550e44635f604dd781fedfdacadb5c2d as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/550e44635f604dd781fedfdacadb5c2d 2024-11-24T20:47:22,835 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/550e44635f604dd781fedfdacadb5c2d, entries=150, sequenceid=320, filesize=12.0 K 2024-11-24T20:47:22,836 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=6.71 KB/6870 for 170c4f62dfd32d7df8a14752b4c9e889 in 1277ms, sequenceid=320, compaction requested=true 2024-11-24T20:47:22,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 170c4f62dfd32d7df8a14752b4c9e889: 2024-11-24T20:47:22,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:22,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-24T20:47:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-24T20:47:22,839 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-24T20:47:22,839 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8900 sec 2024-11-24T20:47:22,840 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.8940 sec 2024-11-24T20:47:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T20:47:23,096 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:47:25,016 INFO [Thread-598 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-598. 2024-11-24T20:47:25,016 DEBUG [Thread-598 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:47:25,016 DEBUG [Thread-598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:25,016 DEBUG [Thread-598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:25,016 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:47:25,047 INFO [Thread-600 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-600. 2024-11-24T20:47:25,047 DEBUG [Thread-600 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:47:25,047 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:25,047 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:25,047 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:47:25,067 INFO [Thread-596 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-596. 2024-11-24T20:47:25,067 DEBUG [Thread-596 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:47:25,067 DEBUG [Thread-596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:25,067 DEBUG [Thread-596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:25,067 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:47:30,870 INFO [Thread-594 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-594. 2024-11-24T20:47:30,870 DEBUG [Thread-594 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:47:30,871 DEBUG [Thread-594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:30,871 DEBUG [Thread-594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:30,871 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:47:30,872 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T20:47:30,872 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 8 2024-11-24T20:47:30,872 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-24T20:47:30,872 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-24T20:47:30,872 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-24T20:47:30,872 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 214 2024-11-24T20:47:30,872 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T20:47:30,872 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3624 2024-11-24T20:47:30,872 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3691 2024-11-24T20:47:30,872 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T20:47:30,872 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1653 2024-11-24T20:47:30,872 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4959 rows 2024-11-24T20:47:30,872 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1654 2024-11-24T20:47:30,872 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4962 rows 2024-11-24T20:47:30,873 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T20:47:30,873 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T20:47:30,873 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:30,873 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:30,873 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:47:30,875 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T20:47:30,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T20:47:30,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-24T20:47:30,881 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732481250880"}]},"ts":"1732481250880"} 2024-11-24T20:47:30,882 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T20:47:30,882 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T20:47:30,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T20:47:30,884 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=170c4f62dfd32d7df8a14752b4c9e889, UNASSIGN}] 2024-11-24T20:47:30,885 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=170c4f62dfd32d7df8a14752b4c9e889, UNASSIGN 2024-11-24T20:47:30,885 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=170c4f62dfd32d7df8a14752b4c9e889, regionState=CLOSING, regionLocation=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:30,886 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=170c4f62dfd32d7df8a14752b4c9e889, UNASSIGN because future has completed 2024-11-24T20:47:30,886 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T20:47:30,887 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291}] 2024-11-24T20:47:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-24T20:47:31,039 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:31,039 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T20:47:31,039 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing 170c4f62dfd32d7df8a14752b4c9e889, disabling compactions & flushes 2024-11-24T20:47:31,039 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:31,039 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:31,039 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. after waiting 0 ms 2024-11-24T20:47:31,039 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:31,039 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing 170c4f62dfd32d7df8a14752b4c9e889 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-24T20:47:31,044 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124af386ee7a7a44b95bf055d893f681e2e_170c4f62dfd32d7df8a14752b4c9e889 is 50, key is test_row_1/A:col10/1732481245066/Put/seqid=0 2024-11-24T20:47:31,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742025_1201 (size=9914) 2024-11-24T20:47:31,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-24T20:47:31,448 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:31,453 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124af386ee7a7a44b95bf055d893f681e2e_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124af386ee7a7a44b95bf055d893f681e2e_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:31,455 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/b289790c822e49658eaf8a37e458357b, store: [table=TestAcidGuarantees family=A region=170c4f62dfd32d7df8a14752b4c9e889] 2024-11-24T20:47:31,455 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/b289790c822e49658eaf8a37e458357b is 175, key is test_row_1/A:col10/1732481245066/Put/seqid=0 2024-11-24T20:47:31,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742026_1202 (size=22561) 2024-11-24T20:47:31,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-24T20:47:31,860 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/b289790c822e49658eaf8a37e458357b 2024-11-24T20:47:31,866 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/bcc10109da194ef1b2efa6f26da256a6 is 50, key is test_row_1/B:col10/1732481245066/Put/seqid=0 2024-11-24T20:47:31,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742027_1203 (size=9857) 2024-11-24T20:47:32,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-24T20:47:32,271 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/bcc10109da194ef1b2efa6f26da256a6 2024-11-24T20:47:32,280 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/9eb966d1d70345e2b756dc897f5a59cd is 50, key is test_row_1/C:col10/1732481245066/Put/seqid=0 2024-11-24T20:47:32,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742028_1204 (size=9857) 2024-11-24T20:47:32,684 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/9eb966d1d70345e2b756dc897f5a59cd 2024-11-24T20:47:32,689 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/A/b289790c822e49658eaf8a37e458357b as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/b289790c822e49658eaf8a37e458357b 2024-11-24T20:47:32,693 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/b289790c822e49658eaf8a37e458357b, entries=100, sequenceid=328, filesize=22.0 K 2024-11-24T20:47:32,694 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/B/bcc10109da194ef1b2efa6f26da256a6 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/bcc10109da194ef1b2efa6f26da256a6 2024-11-24T20:47:32,698 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/bcc10109da194ef1b2efa6f26da256a6, entries=100, sequenceid=328, filesize=9.6 K 2024-11-24T20:47:32,699 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/.tmp/C/9eb966d1d70345e2b756dc897f5a59cd as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/9eb966d1d70345e2b756dc897f5a59cd 2024-11-24T20:47:32,704 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/9eb966d1d70345e2b756dc897f5a59cd, entries=100, sequenceid=328, filesize=9.6 K 2024-11-24T20:47:32,704 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 170c4f62dfd32d7df8a14752b4c9e889 in 1665ms, sequenceid=328, compaction requested=true 2024-11-24T20:47:32,705 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/9995049a8af0418bac5f3ff41bc15e43, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/014ec1a9d9f840dc8afe47d46c5ceab9, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/b3c0657b4af143388ed3a3a7d5912938, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/d81a0b1ca2344113840c20b06dc6e381, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/afea27a9461b4b238744cf245fed946f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/3fc0a8a45f9e48788e303a7b3f442d83, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/8fa838d4a7924c3caa656a64099e794f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/61505695631d4a12a89ce20a456f1c56, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/40b6f82ef8d94a9cb351e7f518d5b0ab, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/95075228b1e346cca3297a0befa451a0, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/3af31d214c5b48219c2f2721098f5cea, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/51c07530f1074930ac8e22a79a82d4b2, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/d2371650466f41849f79f93b6ac5b632, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/319730d0ea7b4110bf7b53d0bb380c7a, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/c9c22cbe8b6d4563a76346080005864c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/cb287ca18f294aa093b31b9d750b8e78, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/4d1ed7e972e3472082be7bc7553796a1, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/eee172fb67864c959a6735024f7917fa] to archive 2024-11-24T20:47:32,706 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T20:47:32,708 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/9995049a8af0418bac5f3ff41bc15e43 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/9995049a8af0418bac5f3ff41bc15e43 2024-11-24T20:47:32,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/014ec1a9d9f840dc8afe47d46c5ceab9 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/014ec1a9d9f840dc8afe47d46c5ceab9 2024-11-24T20:47:32,710 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/b3c0657b4af143388ed3a3a7d5912938 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/b3c0657b4af143388ed3a3a7d5912938 2024-11-24T20:47:32,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/d81a0b1ca2344113840c20b06dc6e381 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/d81a0b1ca2344113840c20b06dc6e381 2024-11-24T20:47:32,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/afea27a9461b4b238744cf245fed946f to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/afea27a9461b4b238744cf245fed946f 2024-11-24T20:47:32,714 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/3fc0a8a45f9e48788e303a7b3f442d83 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/3fc0a8a45f9e48788e303a7b3f442d83 2024-11-24T20:47:32,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/8fa838d4a7924c3caa656a64099e794f to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/8fa838d4a7924c3caa656a64099e794f 2024-11-24T20:47:32,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/61505695631d4a12a89ce20a456f1c56 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/61505695631d4a12a89ce20a456f1c56 2024-11-24T20:47:32,717 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/40b6f82ef8d94a9cb351e7f518d5b0ab to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/40b6f82ef8d94a9cb351e7f518d5b0ab 2024-11-24T20:47:32,718 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/95075228b1e346cca3297a0befa451a0 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/95075228b1e346cca3297a0befa451a0 2024-11-24T20:47:32,720 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/3af31d214c5b48219c2f2721098f5cea to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/3af31d214c5b48219c2f2721098f5cea 2024-11-24T20:47:32,721 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/51c07530f1074930ac8e22a79a82d4b2 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/51c07530f1074930ac8e22a79a82d4b2 2024-11-24T20:47:32,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/d2371650466f41849f79f93b6ac5b632 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/d2371650466f41849f79f93b6ac5b632 2024-11-24T20:47:32,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/319730d0ea7b4110bf7b53d0bb380c7a to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/319730d0ea7b4110bf7b53d0bb380c7a 2024-11-24T20:47:32,724 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/c9c22cbe8b6d4563a76346080005864c to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/c9c22cbe8b6d4563a76346080005864c 2024-11-24T20:47:32,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/cb287ca18f294aa093b31b9d750b8e78 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/cb287ca18f294aa093b31b9d750b8e78 2024-11-24T20:47:32,726 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/4d1ed7e972e3472082be7bc7553796a1 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/4d1ed7e972e3472082be7bc7553796a1 2024-11-24T20:47:32,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/eee172fb67864c959a6735024f7917fa to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/eee172fb67864c959a6735024f7917fa 2024-11-24T20:47:32,729 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/073cc978f2e5470b94a12fcb99656c26, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/7c794d463c214167879880f41b7b2319, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/3dcd852b4dc44289a7bcfecc80818625, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/2eb733e6c6c34c879b22af1e097adcd3, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/fb350b5b5eb64da2bcca7585fad7fb27, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/b6d59cda61ca41ada27c76565e152153, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/f73e78ac1d2c49af9f52070b70967af0, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/3887e781204b46bc840bf0e93fb81799, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/73e52585e63d4cbd84093d2a9d5353be, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/f963b7669c4c438ea8504154537fdf4a, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/451f52bda39a4e27b3ed4946f295c301, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/b40a9ed502c7403e9ca62f12f2530af9, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/7420b22aa10347d3ba66cefcda44cadf, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/a73927c749e8492c8211e2abb25e9ed5, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/b79a93c0c7ac4b859b47291649141c00, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/079698bfafa444b3b21a9937454b0133, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/2bc71dd1b35e4c109749a02ca89f82b9, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/27b6f33b38324898946a329af1c49ea1] to archive 2024-11-24T20:47:32,730 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T20:47:32,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/073cc978f2e5470b94a12fcb99656c26 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/073cc978f2e5470b94a12fcb99656c26 2024-11-24T20:47:32,733 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/7c794d463c214167879880f41b7b2319 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/7c794d463c214167879880f41b7b2319 2024-11-24T20:47:32,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/3dcd852b4dc44289a7bcfecc80818625 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/3dcd852b4dc44289a7bcfecc80818625 2024-11-24T20:47:32,735 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/2eb733e6c6c34c879b22af1e097adcd3 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/2eb733e6c6c34c879b22af1e097adcd3 2024-11-24T20:47:32,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/fb350b5b5eb64da2bcca7585fad7fb27 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/fb350b5b5eb64da2bcca7585fad7fb27 2024-11-24T20:47:32,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/b6d59cda61ca41ada27c76565e152153 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/b6d59cda61ca41ada27c76565e152153 2024-11-24T20:47:32,738 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/f73e78ac1d2c49af9f52070b70967af0 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/f73e78ac1d2c49af9f52070b70967af0 2024-11-24T20:47:32,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/3887e781204b46bc840bf0e93fb81799 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/3887e781204b46bc840bf0e93fb81799 2024-11-24T20:47:32,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/73e52585e63d4cbd84093d2a9d5353be to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/73e52585e63d4cbd84093d2a9d5353be 2024-11-24T20:47:32,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/f963b7669c4c438ea8504154537fdf4a to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/f963b7669c4c438ea8504154537fdf4a 2024-11-24T20:47:32,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/451f52bda39a4e27b3ed4946f295c301 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/451f52bda39a4e27b3ed4946f295c301 2024-11-24T20:47:32,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/b40a9ed502c7403e9ca62f12f2530af9 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/b40a9ed502c7403e9ca62f12f2530af9 2024-11-24T20:47:32,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/7420b22aa10347d3ba66cefcda44cadf to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/7420b22aa10347d3ba66cefcda44cadf 2024-11-24T20:47:32,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/a73927c749e8492c8211e2abb25e9ed5 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/a73927c749e8492c8211e2abb25e9ed5 2024-11-24T20:47:32,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/b79a93c0c7ac4b859b47291649141c00 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/b79a93c0c7ac4b859b47291649141c00 2024-11-24T20:47:32,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/079698bfafa444b3b21a9937454b0133 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/079698bfafa444b3b21a9937454b0133 2024-11-24T20:47:32,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/2bc71dd1b35e4c109749a02ca89f82b9 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/2bc71dd1b35e4c109749a02ca89f82b9 2024-11-24T20:47:32,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/27b6f33b38324898946a329af1c49ea1 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/27b6f33b38324898946a329af1c49ea1 2024-11-24T20:47:32,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/1f2707c7acd543d8b4c7638ad4fba795, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/c049bd44e39c411dba45f02911d1dffa, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/a4ce39dd8a804d85a5d9177aae7f1aa8, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/1fb65818a076478e80f53807e56d460c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/66099b6d42f44f0699441014ecbfdceb, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/e0b21b2505694fa7b1fa221256087aab, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/91dcfd6ad3d44f5b9c522bf7b55eef25, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/c79e80d624c846c98200f3eb980574fb, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/bf7eec729d494bc2b64429575d46abf3, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/45557526582347c69e4ad9c4c2177e52, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/0745b8952e044fa08a65387b09829533, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/65fdbb3128f24e6a898307260e12abf5, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/16641a834e9c4f448a6cb3e42f602bf6, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/3a4ed5497593499aa7cdae367cae7985, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/aa57d140fa7f442aa458d9b3f702b11b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/d5ce1b45ecae41759717c006a0a274c1, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/e4240b4fb93c47f29992ede18ff24ac3, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/ba559abb183248758cac39f3e25ab579] to archive 2024-11-24T20:47:32,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T20:47:32,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/1f2707c7acd543d8b4c7638ad4fba795 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/1f2707c7acd543d8b4c7638ad4fba795 2024-11-24T20:47:32,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/c049bd44e39c411dba45f02911d1dffa to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/c049bd44e39c411dba45f02911d1dffa 2024-11-24T20:47:32,756 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/a4ce39dd8a804d85a5d9177aae7f1aa8 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/a4ce39dd8a804d85a5d9177aae7f1aa8 2024-11-24T20:47:32,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/1fb65818a076478e80f53807e56d460c to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/1fb65818a076478e80f53807e56d460c 2024-11-24T20:47:32,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/66099b6d42f44f0699441014ecbfdceb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/66099b6d42f44f0699441014ecbfdceb 2024-11-24T20:47:32,759 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/e0b21b2505694fa7b1fa221256087aab to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/e0b21b2505694fa7b1fa221256087aab 2024-11-24T20:47:32,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/91dcfd6ad3d44f5b9c522bf7b55eef25 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/91dcfd6ad3d44f5b9c522bf7b55eef25 2024-11-24T20:47:32,761 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/c79e80d624c846c98200f3eb980574fb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/c79e80d624c846c98200f3eb980574fb 2024-11-24T20:47:32,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/bf7eec729d494bc2b64429575d46abf3 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/bf7eec729d494bc2b64429575d46abf3 2024-11-24T20:47:32,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/45557526582347c69e4ad9c4c2177e52 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/45557526582347c69e4ad9c4c2177e52 2024-11-24T20:47:32,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/0745b8952e044fa08a65387b09829533 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/0745b8952e044fa08a65387b09829533 2024-11-24T20:47:32,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/65fdbb3128f24e6a898307260e12abf5 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/65fdbb3128f24e6a898307260e12abf5 2024-11-24T20:47:32,766 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/16641a834e9c4f448a6cb3e42f602bf6 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/16641a834e9c4f448a6cb3e42f602bf6 2024-11-24T20:47:32,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/3a4ed5497593499aa7cdae367cae7985 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/3a4ed5497593499aa7cdae367cae7985 2024-11-24T20:47:32,768 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/aa57d140fa7f442aa458d9b3f702b11b to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/aa57d140fa7f442aa458d9b3f702b11b 2024-11-24T20:47:32,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/d5ce1b45ecae41759717c006a0a274c1 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/d5ce1b45ecae41759717c006a0a274c1 2024-11-24T20:47:32,770 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/e4240b4fb93c47f29992ede18ff24ac3 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/e4240b4fb93c47f29992ede18ff24ac3 2024-11-24T20:47:32,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/ba559abb183248758cac39f3e25ab579 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/ba559abb183248758cac39f3e25ab579 2024-11-24T20:47:32,775 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/recovered.edits/331.seqid, newMaxSeqId=331, maxSeqId=4 2024-11-24T20:47:32,776 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. 2024-11-24T20:47:32,776 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for 170c4f62dfd32d7df8a14752b4c9e889: Waiting for close lock at 1732481251039Running coprocessor pre-close hooks at 1732481251039Disabling compacts and flushes for region at 1732481251039Disabling writes for close at 1732481251039Obtaining lock to block concurrent updates at 1732481251039Preparing flush snapshotting stores in 170c4f62dfd32d7df8a14752b4c9e889 at 1732481251039Finished memstore snapshotting TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732481251039Flushing stores of TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889. at 1732481251040 (+1 ms)Flushing 170c4f62dfd32d7df8a14752b4c9e889/A: creating writer at 1732481251040Flushing mob file 170c4f62dfd32d7df8a14752b4c9e889/A: appending metadata at 1732481251044 (+4 ms)Flushing mob file 170c4f62dfd32d7df8a14752b4c9e889/A: closing flushed file at 1732481251044Flushing 170c4f62dfd32d7df8a14752b4c9e889/A: appending metadata at 1732481251455 (+411 ms)Flushing 170c4f62dfd32d7df8a14752b4c9e889/A: closing flushed file at 1732481251455Flushing 170c4f62dfd32d7df8a14752b4c9e889/B: creating writer at 1732481251863 (+408 ms)Flushing 170c4f62dfd32d7df8a14752b4c9e889/B: appending metadata at 1732481251866 (+3 ms)Flushing 170c4f62dfd32d7df8a14752b4c9e889/B: closing flushed file at 1732481251866Flushing 170c4f62dfd32d7df8a14752b4c9e889/C: creating writer at 1732481252276 (+410 ms)Flushing 170c4f62dfd32d7df8a14752b4c9e889/C: appending metadata at 1732481252280 (+4 ms)Flushing 170c4f62dfd32d7df8a14752b4c9e889/C: closing flushed file at 1732481252280Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@724e11a5: reopening flushed file at 1732481252688 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c604310: reopening flushed file at 1732481252693 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@285ce79b: reopening flushed file at 1732481252698 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 170c4f62dfd32d7df8a14752b4c9e889 in 1665ms, sequenceid=328, compaction requested=true at 1732481252705 (+7 ms)Writing region close event to WAL at 1732481252772 (+67 ms)Running coprocessor post-close hooks at 1732481252776 (+4 ms)Closed at 1732481252776 2024-11-24T20:47:32,779 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed 170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:32,779 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=170c4f62dfd32d7df8a14752b4c9e889, regionState=CLOSED 2024-11-24T20:47:32,780 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 because future has completed 2024-11-24T20:47:32,782 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-24T20:47:32,783 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure 170c4f62dfd32d7df8a14752b4c9e889, server=20b981ef5c27,34625,1732481182291 in 1.8950 sec 2024-11-24T20:47:32,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-24T20:47:32,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=170c4f62dfd32d7df8a14752b4c9e889, UNASSIGN in 1.8990 sec 2024-11-24T20:47:32,786 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-24T20:47:32,786 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9010 sec 2024-11-24T20:47:32,787 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732481252787"}]},"ts":"1732481252787"} 2024-11-24T20:47:32,788 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T20:47:32,788 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T20:47:32,790 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9140 sec 2024-11-24T20:47:33,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-24T20:47:33,026 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:47:33,027 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T20:47:33,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:47:33,030 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:47:33,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-24T20:47:33,032 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:47:33,035 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:33,038 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A, FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B, FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C, FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/recovered.edits] 2024-11-24T20:47:33,041 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/7146399566fc4c848c71613d23ed142d to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/7146399566fc4c848c71613d23ed142d 2024-11-24T20:47:33,042 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/b289790c822e49658eaf8a37e458357b to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/b289790c822e49658eaf8a37e458357b 2024-11-24T20:47:33,044 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/ca68dd45e76648bd88c79e1aec2c2e9f to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/ca68dd45e76648bd88c79e1aec2c2e9f 2024-11-24T20:47:33,046 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/cbd1564ae0534910a164bbcf63477a48 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/A/cbd1564ae0534910a164bbcf63477a48 2024-11-24T20:47:33,049 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/703522dfed194cbfb8c4d56984fdf0bb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/703522dfed194cbfb8c4d56984fdf0bb 2024-11-24T20:47:33,051 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/791ea83d3f3042f89cb2643183644661 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/791ea83d3f3042f89cb2643183644661 2024-11-24T20:47:33,053 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/bcc10109da194ef1b2efa6f26da256a6 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/bcc10109da194ef1b2efa6f26da256a6 2024-11-24T20:47:33,054 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/e1408808f19f4bd1b81c33365b272481 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/B/e1408808f19f4bd1b81c33365b272481 2024-11-24T20:47:33,058 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/550e44635f604dd781fedfdacadb5c2d to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/550e44635f604dd781fedfdacadb5c2d 2024-11-24T20:47:33,060 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/5e6b7bf41b51407593250e6b14831570 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/5e6b7bf41b51407593250e6b14831570 2024-11-24T20:47:33,062 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/9eb966d1d70345e2b756dc897f5a59cd to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/9eb966d1d70345e2b756dc897f5a59cd 2024-11-24T20:47:33,063 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/cd9a5a95d6c54b91bca61a6a04d51174 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/C/cd9a5a95d6c54b91bca61a6a04d51174 2024-11-24T20:47:33,068 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/recovered.edits/331.seqid to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889/recovered.edits/331.seqid 2024-11-24T20:47:33,069 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:33,069 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T20:47:33,070 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T20:47:33,072 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-24T20:47:33,078 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124107924c712a949419a4c07ab1a7bfedd_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124107924c712a949419a4c07ab1a7bfedd_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:33,080 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241e84e71a4136499d972099e0eb6491dd_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241e84e71a4136499d972099e0eb6491dd_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:33,081 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124295d653563f94c2a91e9523c64bbfa81_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124295d653563f94c2a91e9523c64bbfa81_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:33,083 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242e6dcc85d8a343faa13141918a5b051d_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242e6dcc85d8a343faa13141918a5b051d_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:33,084 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242f4a1749d2bb43edb5feea82dac4d9d3_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242f4a1749d2bb43edb5feea82dac4d9d3_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:33,085 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112446adc57f1edb4fabb4b1944c58fb8918_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112446adc57f1edb4fabb4b1944c58fb8918_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:33,087 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244f20bfbebb954128bf211c58e04e8e40_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244f20bfbebb954128bf211c58e04e8e40_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:33,088 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124540df9b4ae3c4931988f0b6e089aabef_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124540df9b4ae3c4931988f0b6e089aabef_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:33,092 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124573976522303420fb449acf993c3b6b8_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124573976522303420fb449acf993c3b6b8_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:33,094 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245f6cc7caaef143ea92a8dda0b9bd4bd4_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245f6cc7caaef143ea92a8dda0b9bd4bd4_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:33,095 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246c6f6a8d6657438e92e1d582cf6da960_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246c6f6a8d6657438e92e1d582cf6da960_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:33,096 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247b262a928b214404a76c1edec80f4dfc_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247b262a928b214404a76c1edec80f4dfc_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:33,098 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124a3ba1689380446a69f1d7bd70d32bab2_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124a3ba1689380446a69f1d7bd70d32bab2_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:33,099 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124a72a39f2473b4e899834fe074f53b52c_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124a72a39f2473b4e899834fe074f53b52c_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:33,100 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124af386ee7a7a44b95bf055d893f681e2e_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124af386ee7a7a44b95bf055d893f681e2e_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:33,102 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bdfb07be1e344e7c9a1c0c5b76043ae3_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bdfb07be1e344e7c9a1c0c5b76043ae3_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:33,104 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bf90a90a1a8145b1a2aca9ba44b3a5fa_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bf90a90a1a8145b1a2aca9ba44b3a5fa_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:33,105 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f2dc7f0b80be42f3816594ce653b1364_170c4f62dfd32d7df8a14752b4c9e889 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f2dc7f0b80be42f3816594ce653b1364_170c4f62dfd32d7df8a14752b4c9e889 2024-11-24T20:47:33,106 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T20:47:33,108 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:47:33,112 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T20:47:33,115 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T20:47:33,116 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:47:33,116 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T20:47:33,117 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732481253117"}]},"ts":"9223372036854775807"} 2024-11-24T20:47:33,120 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T20:47:33,120 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 170c4f62dfd32d7df8a14752b4c9e889, NAME => 'TestAcidGuarantees,,1732481218646.170c4f62dfd32d7df8a14752b4c9e889.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T20:47:33,120 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T20:47:33,121 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732481253120"}]},"ts":"9223372036854775807"} 2024-11-24T20:47:33,123 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T20:47:33,124 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:47:33,125 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 97 msec 2024-11-24T20:47:33,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-24T20:47:33,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T20:47:33,136 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:47:33,150 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239 (was 237) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/cluster_32fb3c48-f3a2-7f88-64a5-636927e4ea42/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639242380_22 at /127.0.0.1:59292 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/cluster_32fb3c48-f3a2-7f88-64a5-636927e4ea42/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=522 (was 554), ProcessCount=11 (was 11), AvailableMemoryMB=10013 (was 10218) 2024-11-24T20:47:33,160 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=239, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=522, ProcessCount=11, AvailableMemoryMB=10011 2024-11-24T20:47:33,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T20:47:33,162 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T20:47:33,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T20:47:33,164 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T20:47:33,164 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:33,164 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-11-24T20:47:33,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-24T20:47:33,166 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T20:47:33,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742029_1205 (size=959) 2024-11-24T20:47:33,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-24T20:47:33,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-24T20:47:33,576 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a 2024-11-24T20:47:33,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742030_1206 (size=53) 2024-11-24T20:47:33,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-24T20:47:33,982 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:47:33,982 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1bb3e8397087a4722761784332990ead, disabling compactions & flushes 2024-11-24T20:47:33,982 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:33,982 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:33,982 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. after waiting 0 ms 2024-11-24T20:47:33,982 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:33,982 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:33,983 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1bb3e8397087a4722761784332990ead: Waiting for close lock at 1732481253982Disabling compacts and flushes for region at 1732481253982Disabling writes for close at 1732481253982Writing region close event to WAL at 1732481253982Closed at 1732481253982 2024-11-24T20:47:33,984 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T20:47:33,984 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732481253984"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732481253984"}]},"ts":"1732481253984"} 2024-11-24T20:47:33,985 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T20:47:33,986 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T20:47:33,987 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732481253986"}]},"ts":"1732481253986"} 2024-11-24T20:47:33,988 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T20:47:33,988 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1bb3e8397087a4722761784332990ead, ASSIGN}] 2024-11-24T20:47:33,989 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1bb3e8397087a4722761784332990ead, ASSIGN 2024-11-24T20:47:33,990 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1bb3e8397087a4722761784332990ead, ASSIGN; state=OFFLINE, location=20b981ef5c27,34625,1732481182291; forceNewPlan=false, retain=false 2024-11-24T20:47:34,140 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=1bb3e8397087a4722761784332990ead, regionState=OPENING, regionLocation=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:34,142 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1bb3e8397087a4722761784332990ead, ASSIGN because future has completed 2024-11-24T20:47:34,142 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291}] 2024-11-24T20:47:34,300 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:34,301 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} 2024-11-24T20:47:34,302 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1bb3e8397087a4722761784332990ead 2024-11-24T20:47:34,302 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:47:34,302 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for 1bb3e8397087a4722761784332990ead 2024-11-24T20:47:34,302 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for 1bb3e8397087a4722761784332990ead 2024-11-24T20:47:34,304 INFO [StoreOpener-1bb3e8397087a4722761784332990ead-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1bb3e8397087a4722761784332990ead 2024-11-24T20:47:34,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-24T20:47:34,306 INFO [StoreOpener-1bb3e8397087a4722761784332990ead-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1bb3e8397087a4722761784332990ead columnFamilyName A 2024-11-24T20:47:34,306 DEBUG [StoreOpener-1bb3e8397087a4722761784332990ead-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:34,306 INFO [StoreOpener-1bb3e8397087a4722761784332990ead-1 {}] regionserver.HStore(327): Store=1bb3e8397087a4722761784332990ead/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:47:34,307 INFO [StoreOpener-1bb3e8397087a4722761784332990ead-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1bb3e8397087a4722761784332990ead 2024-11-24T20:47:34,308 INFO [StoreOpener-1bb3e8397087a4722761784332990ead-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1bb3e8397087a4722761784332990ead columnFamilyName B 2024-11-24T20:47:34,308 DEBUG [StoreOpener-1bb3e8397087a4722761784332990ead-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:34,309 INFO [StoreOpener-1bb3e8397087a4722761784332990ead-1 {}] regionserver.HStore(327): Store=1bb3e8397087a4722761784332990ead/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:47:34,309 INFO [StoreOpener-1bb3e8397087a4722761784332990ead-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1bb3e8397087a4722761784332990ead 2024-11-24T20:47:34,310 INFO [StoreOpener-1bb3e8397087a4722761784332990ead-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1bb3e8397087a4722761784332990ead columnFamilyName C 2024-11-24T20:47:34,310 DEBUG [StoreOpener-1bb3e8397087a4722761784332990ead-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:47:34,310 INFO [StoreOpener-1bb3e8397087a4722761784332990ead-1 {}] regionserver.HStore(327): Store=1bb3e8397087a4722761784332990ead/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:47:34,310 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for 1bb3e8397087a4722761784332990ead 2024-11-24T20:47:34,311 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead 2024-11-24T20:47:34,311 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead 2024-11-24T20:47:34,312 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for 1bb3e8397087a4722761784332990ead 2024-11-24T20:47:34,312 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for 1bb3e8397087a4722761784332990ead 2024-11-24T20:47:34,313 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T20:47:34,314 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for 1bb3e8397087a4722761784332990ead 2024-11-24T20:47:34,316 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T20:47:34,316 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened 1bb3e8397087a4722761784332990ead; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69100156, jitterRate=0.029672563076019287}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T20:47:34,316 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1bb3e8397087a4722761784332990ead 2024-11-24T20:47:34,317 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for 1bb3e8397087a4722761784332990ead: Running coprocessor pre-open hook at 1732481254302Writing region info on filesystem at 1732481254302Initializing all the Stores at 1732481254303 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481254303Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481254304 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481254304Cleaning up temporary data from old regions at 1732481254312 (+8 ms)Running coprocessor post-open hooks at 1732481254316 (+4 ms)Region opened successfully at 1732481254317 (+1 ms) 2024-11-24T20:47:34,318 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., pid=63, masterSystemTime=1732481254294 2024-11-24T20:47:34,320 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=1bb3e8397087a4722761784332990ead, regionState=OPEN, openSeqNum=2, regionLocation=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:34,321 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:34,321 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:34,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 because future has completed 2024-11-24T20:47:34,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-24T20:47:34,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 in 181 msec 2024-11-24T20:47:34,327 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-24T20:47:34,327 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1bb3e8397087a4722761784332990ead, ASSIGN in 337 msec 2024-11-24T20:47:34,327 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T20:47:34,328 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732481254327"}]},"ts":"1732481254327"} 2024-11-24T20:47:34,329 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T20:47:34,330 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T20:47:34,333 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1680 sec 2024-11-24T20:47:35,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-24T20:47:35,315 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:47:35,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43dfb4b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,316 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:47:35,316 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:47:35,317 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:47:35,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:47:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:47:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@664c0265, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:47:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:47:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:35,319 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:47:35,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@347d50aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,320 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:47:35,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:47:35,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:47:35,322 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:47:35,323 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T20:47:35,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 20b981ef5c27,38339,1732481181363 2024-11-24T20:47:35,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@657264a1 2024-11-24T20:47:35,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T20:47:35,325 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44770, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T20:47:35,327 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46ddfbea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,327 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:47:35,327 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:47:35,328 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:47:35,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:47:35,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:47:35,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffb42ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:47:35,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:47:35,329 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:35,329 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44782, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:47:35,330 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68f38845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,331 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fbbb442, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,331 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:47:35,332 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:47:35,333 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:47:35,333 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:47:35,333 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:47:35,333 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b2c52b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,333 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:47:35,333 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:47:35,333 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:35,334 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44794, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:47:35,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3eb8e427, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,335 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@341051a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,336 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:47:35,336 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:47:35,337 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:47:35,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:47:35,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:47:35,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5aed2e60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:47:35,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:47:35,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:35,338 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44806, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:47:35,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4235ef63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,339 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@309aa0fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,339 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:47:35,339 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:47:35,340 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:47:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:47:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:47:35,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e383ad1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:47:35,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:47:35,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:35,342 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:47:35,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70f654b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,343 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6261277b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,343 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:47:35,343 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:47:35,344 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:47:35,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:47:35,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:47:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b02b733, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:47:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:47:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:35,346 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44860, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:47:35,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@618b2268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,347 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@106f6d84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,347 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:47:35,347 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:47:35,348 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:47:35,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:47:35,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:47:35,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@338db323, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:47:35,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:47:35,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:35,349 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44880, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:47:35,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e54d28c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,352 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a00a41f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,352 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:47:35,353 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:47:35,354 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:47:35,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:47:35,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:47:35,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4009fd9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:47:35,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:47:35,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:35,355 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44902, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:47:35,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c2addff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,357 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a680e8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,357 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:47:35,357 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:47:35,358 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:47:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:47:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:47:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df924b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:47:35,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:47:35,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:35,360 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44912, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:47:35,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53fa97c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,361 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@292fd0b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,361 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:47:35,362 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:47:35,362 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:47:35,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:47:35,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:47:35,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8ad9f3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:47:35,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:47:35,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:35,364 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44934, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:47:35,365 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fd1995e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,366 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@710fcbd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,366 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:47:35,366 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:47:35,367 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:47:35,367 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:47:35,367 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:47:35,367 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b1b90e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,367 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:47:35,368 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:47:35,368 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:35,369 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:47:35,369 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e833343, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:47:35,370 DEBUG [Thread-1002 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:47:35,370 DEBUG [Thread-1004 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:47:35,371 DEBUG [Thread-985 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:47:35,371 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:47:35,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:47:35,371 DEBUG [Thread-991 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:47:35,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:47:35,371 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:47:35,371 DEBUG [Thread-1000 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:47:35,371 DEBUG [Thread-996 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:47:35,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:47:35,372 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:47:35,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:47:35,373 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:47:35,373 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:47:35,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:47:35,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:47:35,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:47:35,373 DEBUG [Thread-987 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:47:35,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:47:35,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:47:35,373 DEBUG [Thread-993 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:47:35,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:47:35,374 DEBUG [Thread-989 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:47:35,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:47:35,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-24T20:47:35,374 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42250, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:47:35,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:47:35,375 DEBUG [Thread-998 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:47:35,375 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:47:35,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:47:35,375 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:47:35,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:47:35,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2] 2024-11-24T20:47:35,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T20:47:35,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2] 2024-11-24T20:47:35,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:47:35,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:47:35,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2] 2024-11-24T20:47:35,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:47:35,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:47:35,378 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:47:35,378 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:47:35,379 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42302, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:47:35,379 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:47:35,379 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:47:35,380 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:47:35,380 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:47:35,380 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:47:35,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2] 2024-11-24T20:47:35,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2] 2024-11-24T20:47:35,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2] 2024-11-24T20:47:35,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2] 2024-11-24T20:47:35,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2] 2024-11-24T20:47:35,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2] 2024-11-24T20:47:35,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2] 2024-11-24T20:47:35,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 1bb3e8397087a4722761784332990ead 2024-11-24T20:47:35,389 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:47:35,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1bb3e8397087a4722761784332990ead 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T20:47:35,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/d3658e55f4784d74b6219e6b4368b58f is 50, key is test_row_0/A:col10/1732481255387/Put/seqid=0 2024-11-24T20:47:35,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742031_1207 (size=14341) 2024-11-24T20:47:35,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:35,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481315419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:35,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:35,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:35,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:35,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/d3658e55f4784d74b6219e6b4368b58f 2024-11-24T20:47:35,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:35,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481315424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:35,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:35,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:35,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:35,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:35,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:35,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481315435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:35,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1732481315432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:35,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:35,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:35,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:35,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:35,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:35,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:35,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:35,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481315441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:35,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:35,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:35,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:35,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/e2947a2a81e144b8b15133e80dc51f76 is 50, key is test_row_0/B:col10/1732481255387/Put/seqid=0 2024-11-24T20:47:35,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T20:47:35,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742032_1208 (size=12001) 2024-11-24T20:47:35,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T20:47:35,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:35,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:35,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:35,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:35,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:35,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:35,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:35,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481315536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:35,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:35,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:35,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:35,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:35,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481315555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:35,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:35,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481315555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:35,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:35,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:35,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481315555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:35,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:35,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:35,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:35,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1732481315555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:35,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:35,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:35,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:35,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:35,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:35,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:35,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:35,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:35,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:35,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T20:47:35,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:35,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:35,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:35,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:35,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T20:47:35,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:35,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481315745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:35,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:35,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:35,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:35,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481315765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:35,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:35,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:35,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:35,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481315765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:35,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:35,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:35,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:35,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481315765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:35,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:35,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:35,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:35,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1732481315765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:35,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:35,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:35,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:35,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T20:47:35,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:35,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:35,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:35,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:35,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:35,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/e2947a2a81e144b8b15133e80dc51f76 2024-11-24T20:47:35,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/084cf5f2ee664988b479c3b2eecb9339 is 50, key is test_row_0/C:col10/1732481255387/Put/seqid=0 2024-11-24T20:47:35,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T20:47:35,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:35,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:35,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:35,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:35,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:36,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T20:47:36,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742033_1209 (size=12001) 2024-11-24T20:47:36,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:36,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481316055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:36,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:36,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:36,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481316076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:36,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:36,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:36,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481316076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:36,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:36,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:36,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481316076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:36,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:36,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1732481316076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:36,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:36,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T20:47:36,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:36,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:36,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:36,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:36,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:36,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:36,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T20:47:36,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:36,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:36,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:36,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:36,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:36,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:36,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/084cf5f2ee664988b479c3b2eecb9339 2024-11-24T20:47:36,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/d3658e55f4784d74b6219e6b4368b58f as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/d3658e55f4784d74b6219e6b4368b58f 2024-11-24T20:47:36,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/d3658e55f4784d74b6219e6b4368b58f, entries=200, sequenceid=13, filesize=14.0 K 2024-11-24T20:47:36,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/e2947a2a81e144b8b15133e80dc51f76 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/e2947a2a81e144b8b15133e80dc51f76 2024-11-24T20:47:36,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/e2947a2a81e144b8b15133e80dc51f76, entries=150, sequenceid=13, filesize=11.7 K 2024-11-24T20:47:36,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/084cf5f2ee664988b479c3b2eecb9339 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/084cf5f2ee664988b479c3b2eecb9339 2024-11-24T20:47:36,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/084cf5f2ee664988b479c3b2eecb9339, entries=150, sequenceid=13, filesize=11.7 K 2024-11-24T20:47:36,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 1bb3e8397087a4722761784332990ead in 1052ms, sequenceid=13, compaction requested=false 2024-11-24T20:47:36,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:36,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T20:47:36,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:36,457 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 1bb3e8397087a4722761784332990ead 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-24T20:47:36,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/fbec8a44dfb34d9294a29fa9950a789b is 50, key is test_row_0/A:col10/1732481255428/Put/seqid=0 2024-11-24T20:47:36,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742034_1210 (size=12001) 2024-11-24T20:47:36,479 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/fbec8a44dfb34d9294a29fa9950a789b 2024-11-24T20:47:36,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/e3a24f74823b49489c9ea82ac8f6c02b is 50, key is test_row_0/B:col10/1732481255428/Put/seqid=0 2024-11-24T20:47:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T20:47:36,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742035_1211 (size=12001) 2024-11-24T20:47:36,528 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/e3a24f74823b49489c9ea82ac8f6c02b 2024-11-24T20:47:36,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/119a666dc3994198b7cfdab4cd457350 is 50, key is test_row_0/C:col10/1732481255428/Put/seqid=0 2024-11-24T20:47:36,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742036_1212 (size=12001) 2024-11-24T20:47:36,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 1bb3e8397087a4722761784332990ead 2024-11-24T20:47:36,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:36,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:36,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481316587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:36,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:36,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:36,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481316591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481316591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:36,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:36,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481316592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:36,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:36,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:36,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1732481316595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:36,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:36,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:36,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:36,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481316695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:36,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:36,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:36,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481316705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:36,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:36,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481316705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:36,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:36,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:36,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481316705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:36,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:36,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:36,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:36,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481316912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:36,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:36,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481316915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:36,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:36,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481316915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:36,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:36,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:36,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:36,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481316915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:36,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:36,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:36,969 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/119a666dc3994198b7cfdab4cd457350 2024-11-24T20:47:36,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/fbec8a44dfb34d9294a29fa9950a789b as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/fbec8a44dfb34d9294a29fa9950a789b 2024-11-24T20:47:36,994 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/fbec8a44dfb34d9294a29fa9950a789b, entries=150, sequenceid=38, filesize=11.7 K 2024-11-24T20:47:36,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/e3a24f74823b49489c9ea82ac8f6c02b as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/e3a24f74823b49489c9ea82ac8f6c02b 2024-11-24T20:47:37,005 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/e3a24f74823b49489c9ea82ac8f6c02b, entries=150, sequenceid=38, filesize=11.7 K 2024-11-24T20:47:37,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/119a666dc3994198b7cfdab4cd457350 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/119a666dc3994198b7cfdab4cd457350 2024-11-24T20:47:37,012 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/119a666dc3994198b7cfdab4cd457350, entries=150, sequenceid=38, filesize=11.7 K 2024-11-24T20:47:37,013 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 1bb3e8397087a4722761784332990ead in 556ms, sequenceid=38, compaction requested=false 2024-11-24T20:47:37,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:37,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:37,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-24T20:47:37,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-24T20:47:37,018 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-24T20:47:37,018 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6350 sec 2024-11-24T20:47:37,020 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.6470 sec 2024-11-24T20:47:37,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 1bb3e8397087a4722761784332990ead 2024-11-24T20:47:37,226 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:47:37,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1bb3e8397087a4722761784332990ead 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-24T20:47:37,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/a30faf1ff6be4abf93a6695892c0d807 is 50, key is test_row_0/A:col10/1732481256585/Put/seqid=0 2024-11-24T20:47:37,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742037_1213 (size=12001) 2024-11-24T20:47:37,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/a30faf1ff6be4abf93a6695892c0d807 2024-11-24T20:47:37,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481317263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:37,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481317264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:37,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481317265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:37,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481317265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:37,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:37,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:37,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:37,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:37,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/feec15f5a8c54cf4a72910f558a5fe5b is 50, key is test_row_0/B:col10/1732481256585/Put/seqid=0 2024-11-24T20:47:37,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742038_1214 (size=12001) 2024-11-24T20:47:37,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/feec15f5a8c54cf4a72910f558a5fe5b 2024-11-24T20:47:37,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/883afc54381c4b7893ff01464d81eec2 is 50, key is test_row_0/C:col10/1732481256585/Put/seqid=0 2024-11-24T20:47:37,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742039_1215 (size=12001) 2024-11-24T20:47:37,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481317375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:37,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481317375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:37,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481317376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:37,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481317376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:37,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T20:47:37,525 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:47:37,526 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:47:37,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-24T20:47:37,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T20:47:37,527 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:47:37,528 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:47:37,528 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:47:37,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:37,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481317585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:37,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:37,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:37,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481317585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:37,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481317585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:37,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:37,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:37,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:37,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481317585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:37,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:37,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:37,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:37,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1732481317605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:37,609 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_1' in 1bb3e8397087a4722761784332990ead of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:47:37,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:37,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T20:47:37,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-24T20:47:37,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:37,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:37,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:37,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:37,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:37,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:37,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/883afc54381c4b7893ff01464d81eec2 2024-11-24T20:47:37,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/a30faf1ff6be4abf93a6695892c0d807 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/a30faf1ff6be4abf93a6695892c0d807 2024-11-24T20:47:37,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/a30faf1ff6be4abf93a6695892c0d807, entries=150, sequenceid=52, filesize=11.7 K 2024-11-24T20:47:37,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/feec15f5a8c54cf4a72910f558a5fe5b as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/feec15f5a8c54cf4a72910f558a5fe5b 2024-11-24T20:47:37,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/feec15f5a8c54cf4a72910f558a5fe5b, entries=150, sequenceid=52, filesize=11.7 K 2024-11-24T20:47:37,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/883afc54381c4b7893ff01464d81eec2 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/883afc54381c4b7893ff01464d81eec2 2024-11-24T20:47:37,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/883afc54381c4b7893ff01464d81eec2, entries=150, sequenceid=52, filesize=11.7 K 2024-11-24T20:47:37,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 1bb3e8397087a4722761784332990ead in 541ms, sequenceid=52, compaction requested=true 2024-11-24T20:47:37,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:37,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1bb3e8397087a4722761784332990ead:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:47:37,768 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:47:37,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:37,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1bb3e8397087a4722761784332990ead:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:47:37,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:37,768 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:47:37,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1bb3e8397087a4722761784332990ead:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:47:37,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:47:37,769 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:47:37,769 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 1bb3e8397087a4722761784332990ead/B is initiating minor compaction (all files) 2024-11-24T20:47:37,770 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1bb3e8397087a4722761784332990ead/B in TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:37,770 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/e2947a2a81e144b8b15133e80dc51f76, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/e3a24f74823b49489c9ea82ac8f6c02b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/feec15f5a8c54cf4a72910f558a5fe5b] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp, totalSize=35.2 K 2024-11-24T20:47:37,771 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting e2947a2a81e144b8b15133e80dc51f76, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732481255387 2024-11-24T20:47:37,771 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting e3a24f74823b49489c9ea82ac8f6c02b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732481255415 2024-11-24T20:47:37,773 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting feec15f5a8c54cf4a72910f558a5fe5b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732481256585 2024-11-24T20:47:37,774 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:47:37,774 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 1bb3e8397087a4722761784332990ead/A is initiating minor compaction (all files) 2024-11-24T20:47:37,774 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1bb3e8397087a4722761784332990ead/A in TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:37,775 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/d3658e55f4784d74b6219e6b4368b58f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/fbec8a44dfb34d9294a29fa9950a789b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/a30faf1ff6be4abf93a6695892c0d807] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp, totalSize=37.4 K 2024-11-24T20:47:37,776 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3658e55f4784d74b6219e6b4368b58f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732481255385 2024-11-24T20:47:37,778 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbec8a44dfb34d9294a29fa9950a789b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732481255415 2024-11-24T20:47:37,779 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting a30faf1ff6be4abf93a6695892c0d807, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732481256585 2024-11-24T20:47:37,806 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1bb3e8397087a4722761784332990ead#B#compaction#176 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:37,807 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/cb910dba1e54481bbf10f75a2bcafcd4 is 50, key is test_row_0/B:col10/1732481256585/Put/seqid=0 2024-11-24T20:47:37,810 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1bb3e8397087a4722761784332990ead#A#compaction#177 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:37,811 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/da0e223ebc9a4c54b5e46104dcbda621 is 50, key is test_row_0/A:col10/1732481256585/Put/seqid=0 2024-11-24T20:47:37,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-24T20:47:37,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:37,834 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 1bb3e8397087a4722761784332990ead 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-24T20:47:37,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742040_1216 (size=12104) 2024-11-24T20:47:37,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T20:47:37,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742041_1217 (size=12104) 2024-11-24T20:47:37,850 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/cb910dba1e54481bbf10f75a2bcafcd4 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/cb910dba1e54481bbf10f75a2bcafcd4 2024-11-24T20:47:37,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/18172e16ebd145c9842b515020eca86f is 50, key is test_row_0/A:col10/1732481257263/Put/seqid=0 2024-11-24T20:47:37,865 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1bb3e8397087a4722761784332990ead/B of 1bb3e8397087a4722761784332990ead into cb910dba1e54481bbf10f75a2bcafcd4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:37,865 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:37,865 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., storeName=1bb3e8397087a4722761784332990ead/B, priority=13, startTime=1732481257768; duration=0sec 2024-11-24T20:47:37,865 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:47:37,865 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1bb3e8397087a4722761784332990ead:B 2024-11-24T20:47:37,865 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:47:37,866 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/da0e223ebc9a4c54b5e46104dcbda621 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/da0e223ebc9a4c54b5e46104dcbda621 2024-11-24T20:47:37,866 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:47:37,866 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 1bb3e8397087a4722761784332990ead/C is initiating minor compaction (all files) 2024-11-24T20:47:37,867 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1bb3e8397087a4722761784332990ead/C in TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:37,867 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/084cf5f2ee664988b479c3b2eecb9339, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/119a666dc3994198b7cfdab4cd457350, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/883afc54381c4b7893ff01464d81eec2] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp, totalSize=35.2 K 2024-11-24T20:47:37,867 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 084cf5f2ee664988b479c3b2eecb9339, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732481255387 2024-11-24T20:47:37,868 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 119a666dc3994198b7cfdab4cd457350, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732481255415 2024-11-24T20:47:37,869 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 883afc54381c4b7893ff01464d81eec2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732481256585 2024-11-24T20:47:37,877 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1bb3e8397087a4722761784332990ead/A of 1bb3e8397087a4722761784332990ead into da0e223ebc9a4c54b5e46104dcbda621(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:37,877 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:37,877 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., storeName=1bb3e8397087a4722761784332990ead/A, priority=13, startTime=1732481257768; duration=0sec 2024-11-24T20:47:37,877 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:37,878 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1bb3e8397087a4722761784332990ead:A 2024-11-24T20:47:37,883 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1bb3e8397087a4722761784332990ead#C#compaction#179 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:37,883 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/aed2954fa4184f828b9372f1198ce7cd is 50, key is test_row_0/C:col10/1732481256585/Put/seqid=0 2024-11-24T20:47:37,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742042_1218 (size=12001) 2024-11-24T20:47:37,887 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/18172e16ebd145c9842b515020eca86f 2024-11-24T20:47:37,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 1bb3e8397087a4722761784332990ead 2024-11-24T20:47:37,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:37,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/659cc4d46a12496fad48dfeecd580eb0 is 50, key is test_row_0/B:col10/1732481257263/Put/seqid=0 2024-11-24T20:47:37,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742043_1219 (size=12104) 2024-11-24T20:47:37,913 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/aed2954fa4184f828b9372f1198ce7cd as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/aed2954fa4184f828b9372f1198ce7cd 2024-11-24T20:47:37,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742044_1220 (size=12001) 2024-11-24T20:47:37,937 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1bb3e8397087a4722761784332990ead/C of 1bb3e8397087a4722761784332990ead into aed2954fa4184f828b9372f1198ce7cd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:37,937 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:37,937 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., storeName=1bb3e8397087a4722761784332990ead/C, priority=13, startTime=1732481257768; duration=0sec 2024-11-24T20:47:37,937 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:37,937 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1bb3e8397087a4722761784332990ead:C 2024-11-24T20:47:37,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:37,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481317938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:37,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:37,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481317939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:37,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:37,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:37,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:37,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:37,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481317945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:37,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481317944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:37,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:37,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:37,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:38,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481318055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:38,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481318055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:38,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481318055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:38,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481318055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:38,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:38,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T20:47:38,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:38,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481318265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:38,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:38,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:38,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481318265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:38,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:38,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481318265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:38,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:38,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481318265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:38,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:38,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:38,313 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T20:47:38,330 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/659cc4d46a12496fad48dfeecd580eb0 2024-11-24T20:47:38,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/5d532df640384b549c7c193acbc8a129 is 50, key is test_row_0/C:col10/1732481257263/Put/seqid=0 2024-11-24T20:47:38,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742045_1221 (size=12001) 2024-11-24T20:47:38,368 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/5d532df640384b549c7c193acbc8a129 2024-11-24T20:47:38,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/18172e16ebd145c9842b515020eca86f as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/18172e16ebd145c9842b515020eca86f 2024-11-24T20:47:38,381 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/18172e16ebd145c9842b515020eca86f, entries=150, sequenceid=75, filesize=11.7 K 2024-11-24T20:47:38,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/659cc4d46a12496fad48dfeecd580eb0 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/659cc4d46a12496fad48dfeecd580eb0 2024-11-24T20:47:38,389 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/659cc4d46a12496fad48dfeecd580eb0, entries=150, sequenceid=75, filesize=11.7 K 2024-11-24T20:47:38,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/5d532df640384b549c7c193acbc8a129 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/5d532df640384b549c7c193acbc8a129 2024-11-24T20:47:38,396 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/5d532df640384b549c7c193acbc8a129, entries=150, sequenceid=75, filesize=11.7 K 2024-11-24T20:47:38,397 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 1bb3e8397087a4722761784332990ead in 562ms, sequenceid=75, compaction requested=false 2024-11-24T20:47:38,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:38,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:38,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-24T20:47:38,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-24T20:47:38,402 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-24T20:47:38,402 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 870 msec 2024-11-24T20:47:38,403 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 876 msec 2024-11-24T20:47:38,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 1bb3e8397087a4722761784332990ead 2024-11-24T20:47:38,576 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:47:38,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1bb3e8397087a4722761784332990ead 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-24T20:47:38,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/9692348eae134b208e34555a3a609bf7 is 50, key is test_row_0/A:col10/1732481257943/Put/seqid=0 2024-11-24T20:47:38,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742046_1222 (size=14341) 2024-11-24T20:47:38,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481318599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:38,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481318599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:38,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:38,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:38,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:38,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481318603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:38,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:38,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:38,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481318604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:38,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T20:47:38,665 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:47:38,665 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:47:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-24T20:47:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-24T20:47:38,667 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:47:38,667 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:47:38,667 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:47:38,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481318715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:38,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:38,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481318715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:38,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:38,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:38,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481318715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:38,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:38,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:38,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481318716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:38,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:38,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-24T20:47:38,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-24T20:47:38,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:38,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:38,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:38,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:38,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:38,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:38,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:38,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481318925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:38,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:38,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481318925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:38,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481318925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:38,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481318925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:38,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-24T20:47:38,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:38,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:38,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:38,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:38,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:38,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:38,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-24T20:47:38,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/9692348eae134b208e34555a3a609bf7 2024-11-24T20:47:39,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/8fde4f32c3c44b7ab4c7df9124ff612d is 50, key is test_row_0/B:col10/1732481257943/Put/seqid=0 2024-11-24T20:47:39,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742047_1223 (size=12001) 2024-11-24T20:47:39,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-24T20:47:39,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:39,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:39,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:39,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:39,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:39,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:39,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:39,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481319235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:39,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:39,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:39,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:39,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:39,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481319235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:39,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:39,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481319235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:39,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:39,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:39,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:39,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:39,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:39,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481319235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:39,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:39,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:39,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:39,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:39,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:39,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-24T20:47:39,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:39,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:39,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:39,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:39,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:39,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:39,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-24T20:47:39,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/8fde4f32c3c44b7ab4c7df9124ff612d 2024-11-24T20:47:39,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/3ed03bd1a58848bb8b499c1ada852220 is 50, key is test_row_0/C:col10/1732481257943/Put/seqid=0 2024-11-24T20:47:39,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742048_1224 (size=12001) 2024-11-24T20:47:39,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-24T20:47:39,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:39,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:39,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:39,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:39,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:39,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:39,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-24T20:47:39,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:39,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:39,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:39,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:39,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:39,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:39,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:39,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1732481319625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:39,628 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_1' in 1bb3e8397087a4722761784332990ead of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:47:39,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:39,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:39,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:39,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-24T20:47:39,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:39,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:39,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:39,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:39,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:39,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:39,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481319745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:39,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:39,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:39,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:39,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481319745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:39,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:39,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:39,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481319746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:39,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:39,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:39,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:39,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:39,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:39,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481319755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:39,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:39,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:39,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:39,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-24T20:47:39,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/3ed03bd1a58848bb8b499c1ada852220 2024-11-24T20:47:39,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/9692348eae134b208e34555a3a609bf7 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/9692348eae134b208e34555a3a609bf7 2024-11-24T20:47:39,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/9692348eae134b208e34555a3a609bf7, entries=200, sequenceid=93, filesize=14.0 K 2024-11-24T20:47:39,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/8fde4f32c3c44b7ab4c7df9124ff612d as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/8fde4f32c3c44b7ab4c7df9124ff612d 2024-11-24T20:47:39,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/8fde4f32c3c44b7ab4c7df9124ff612d, entries=150, sequenceid=93, filesize=11.7 K 2024-11-24T20:47:39,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/3ed03bd1a58848bb8b499c1ada852220 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/3ed03bd1a58848bb8b499c1ada852220 2024-11-24T20:47:39,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/3ed03bd1a58848bb8b499c1ada852220, entries=150, sequenceid=93, filesize=11.7 K 2024-11-24T20:47:39,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 1bb3e8397087a4722761784332990ead in 1262ms, sequenceid=93, compaction requested=true 2024-11-24T20:47:39,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:39,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1bb3e8397087a4722761784332990ead:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:47:39,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:39,838 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:47:39,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1bb3e8397087a4722761784332990ead:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:47:39,839 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:47:39,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:39,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1bb3e8397087a4722761784332990ead:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:47:39,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:47:39,840 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:47:39,840 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 1bb3e8397087a4722761784332990ead/B is initiating minor compaction (all files) 2024-11-24T20:47:39,840 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1bb3e8397087a4722761784332990ead/B in TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:39,840 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/cb910dba1e54481bbf10f75a2bcafcd4, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/659cc4d46a12496fad48dfeecd580eb0, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/8fde4f32c3c44b7ab4c7df9124ff612d] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp, totalSize=35.3 K 2024-11-24T20:47:39,840 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:47:39,840 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 1bb3e8397087a4722761784332990ead/A is initiating minor compaction (all files) 2024-11-24T20:47:39,840 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1bb3e8397087a4722761784332990ead/A in TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:39,840 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/da0e223ebc9a4c54b5e46104dcbda621, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/18172e16ebd145c9842b515020eca86f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/9692348eae134b208e34555a3a609bf7] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp, totalSize=37.5 K 2024-11-24T20:47:39,841 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting da0e223ebc9a4c54b5e46104dcbda621, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732481256585 2024-11-24T20:47:39,841 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting cb910dba1e54481bbf10f75a2bcafcd4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732481256585 2024-11-24T20:47:39,841 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18172e16ebd145c9842b515020eca86f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732481257256 2024-11-24T20:47:39,841 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 659cc4d46a12496fad48dfeecd580eb0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732481257256 2024-11-24T20:47:39,842 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fde4f32c3c44b7ab4c7df9124ff612d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732481257937 2024-11-24T20:47:39,842 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9692348eae134b208e34555a3a609bf7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732481257937 2024-11-24T20:47:39,851 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1bb3e8397087a4722761784332990ead#A#compaction#185 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:39,851 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1bb3e8397087a4722761784332990ead#B#compaction#186 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:39,851 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/2c3f228644ff498babadbd0491d67800 is 50, key is test_row_0/A:col10/1732481257943/Put/seqid=0 2024-11-24T20:47:39,851 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/c9c74ce5ad194289bf4775f1d708e99a is 50, key is test_row_0/B:col10/1732481257943/Put/seqid=0 2024-11-24T20:47:39,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742050_1226 (size=12207) 2024-11-24T20:47:39,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742049_1225 (size=12207) 2024-11-24T20:47:39,875 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/2c3f228644ff498babadbd0491d67800 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/2c3f228644ff498babadbd0491d67800 2024-11-24T20:47:39,881 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1bb3e8397087a4722761784332990ead/A of 1bb3e8397087a4722761784332990ead into 2c3f228644ff498babadbd0491d67800(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:39,881 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:39,881 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., storeName=1bb3e8397087a4722761784332990ead/A, priority=13, startTime=1732481259838; duration=0sec 2024-11-24T20:47:39,881 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:47:39,881 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1bb3e8397087a4722761784332990ead:A 2024-11-24T20:47:39,881 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:47:39,883 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:47:39,883 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 1bb3e8397087a4722761784332990ead/C is initiating minor compaction (all files) 2024-11-24T20:47:39,883 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1bb3e8397087a4722761784332990ead/C in TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:39,883 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/aed2954fa4184f828b9372f1198ce7cd, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/5d532df640384b549c7c193acbc8a129, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/3ed03bd1a58848bb8b499c1ada852220] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp, totalSize=35.3 K 2024-11-24T20:47:39,883 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting aed2954fa4184f828b9372f1198ce7cd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732481256585 2024-11-24T20:47:39,883 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d532df640384b549c7c193acbc8a129, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732481257256 2024-11-24T20:47:39,884 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ed03bd1a58848bb8b499c1ada852220, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732481257937 2024-11-24T20:47:39,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-24T20:47:39,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:39,890 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 1bb3e8397087a4722761784332990ead 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-24T20:47:39,891 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1bb3e8397087a4722761784332990ead#C#compaction#187 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:39,892 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/bc5433afd90c49d6b8219c9fa159d0e6 is 50, key is test_row_0/C:col10/1732481257943/Put/seqid=0 2024-11-24T20:47:39,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/72bd01b05e4347f8866dfaa28dd3d3c7 is 50, key is test_row_0/A:col10/1732481258602/Put/seqid=0 2024-11-24T20:47:39,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742051_1227 (size=12207) 2024-11-24T20:47:39,907 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/bc5433afd90c49d6b8219c9fa159d0e6 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/bc5433afd90c49d6b8219c9fa159d0e6 2024-11-24T20:47:39,913 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1bb3e8397087a4722761784332990ead/C of 1bb3e8397087a4722761784332990ead into bc5433afd90c49d6b8219c9fa159d0e6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:39,913 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:39,913 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., storeName=1bb3e8397087a4722761784332990ead/C, priority=13, startTime=1732481259839; duration=0sec 2024-11-24T20:47:39,913 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:39,913 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1bb3e8397087a4722761784332990ead:C 2024-11-24T20:47:39,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742052_1228 (size=12001) 2024-11-24T20:47:40,275 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/c9c74ce5ad194289bf4775f1d708e99a as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/c9c74ce5ad194289bf4775f1d708e99a 2024-11-24T20:47:40,280 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1bb3e8397087a4722761784332990ead/B of 1bb3e8397087a4722761784332990ead into c9c74ce5ad194289bf4775f1d708e99a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:40,280 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:40,280 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., storeName=1bb3e8397087a4722761784332990ead/B, priority=13, startTime=1732481259838; duration=0sec 2024-11-24T20:47:40,280 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:40,280 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1bb3e8397087a4722761784332990ead:B 2024-11-24T20:47:40,316 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/72bd01b05e4347f8866dfaa28dd3d3c7 2024-11-24T20:47:40,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/03d8240044ab46e5be9ac7af8a985692 is 50, key is test_row_0/B:col10/1732481258602/Put/seqid=0 2024-11-24T20:47:40,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742053_1229 (size=12001) 2024-11-24T20:47:40,730 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/03d8240044ab46e5be9ac7af8a985692 2024-11-24T20:47:40,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/4d70fd7745614ae485465c8f672616b5 is 50, key is test_row_0/C:col10/1732481258602/Put/seqid=0 2024-11-24T20:47:40,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742054_1230 (size=12001) 2024-11-24T20:47:40,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 1bb3e8397087a4722761784332990ead 2024-11-24T20:47:40,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:40,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:40,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481320765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:40,769 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_2' in 1bb3e8397087a4722761784332990ead of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:47:40,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:40,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:40,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:40,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:40,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481320768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:40,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:40,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481320769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:40,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:40,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481320769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:40,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-24T20:47:40,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:40,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481320875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:40,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:40,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:40,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481320875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:40,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:40,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:40,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:40,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:40,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:40,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:40,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481320875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:40,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:40,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:40,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:41,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:41,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481321085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:41,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:41,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481321085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:41,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:41,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:41,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:41,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:41,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:41,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:41,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:41,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481321085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:41,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:41,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:41,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:41,144 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/4d70fd7745614ae485465c8f672616b5 2024-11-24T20:47:41,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/72bd01b05e4347f8866dfaa28dd3d3c7 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/72bd01b05e4347f8866dfaa28dd3d3c7 2024-11-24T20:47:41,159 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/72bd01b05e4347f8866dfaa28dd3d3c7, entries=150, sequenceid=116, filesize=11.7 K 2024-11-24T20:47:41,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/03d8240044ab46e5be9ac7af8a985692 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/03d8240044ab46e5be9ac7af8a985692 2024-11-24T20:47:41,164 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/03d8240044ab46e5be9ac7af8a985692, entries=150, sequenceid=116, filesize=11.7 K 2024-11-24T20:47:41,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/4d70fd7745614ae485465c8f672616b5 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/4d70fd7745614ae485465c8f672616b5 2024-11-24T20:47:41,169 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/4d70fd7745614ae485465c8f672616b5, entries=150, sequenceid=116, filesize=11.7 K 2024-11-24T20:47:41,170 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 1bb3e8397087a4722761784332990ead in 1280ms, sequenceid=116, compaction requested=false 2024-11-24T20:47:41,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:41,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:41,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-24T20:47:41,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-24T20:47:41,173 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-24T20:47:41,173 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5040 sec 2024-11-24T20:47:41,175 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.5090 sec 2024-11-24T20:47:41,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 1bb3e8397087a4722761784332990ead 2024-11-24T20:47:41,397 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:47:41,397 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1bb3e8397087a4722761784332990ead 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-24T20:47:41,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/0b4d31beebcb4e23aa2ecee9582bed3d is 50, key is test_row_0/A:col10/1732481261396/Put/seqid=0 2024-11-24T20:47:41,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742055_1231 (size=16881) 2024-11-24T20:47:41,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:41,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481321445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:41,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:41,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:41,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:41,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:41,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481321445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:41,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:41,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:41,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:41,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:41,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481321449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:41,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:41,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:41,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:41,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481321555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:41,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:41,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481321555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:41,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:41,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:41,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:41,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:41,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481321555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:41,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:41,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481321765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:41,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:41,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:41,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:41,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:41,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481321765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:41,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:41,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:41,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481321765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:41,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:41,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:41,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:41,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:41,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:41,807 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/0b4d31beebcb4e23aa2ecee9582bed3d 2024-11-24T20:47:41,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/8e38a712026d452195a06ffe85872191 is 50, key is test_row_0/B:col10/1732481261396/Put/seqid=0 2024-11-24T20:47:41,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742056_1232 (size=12101) 2024-11-24T20:47:42,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:42,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481322075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:42,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:42,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481322075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:42,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:42,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481322075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:42,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:42,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/8e38a712026d452195a06ffe85872191 2024-11-24T20:47:42,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/66fe8e73b5ba47728c819d0d728bd2bb is 50, key is test_row_0/C:col10/1732481261396/Put/seqid=0 2024-11-24T20:47:42,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742057_1233 (size=12101) 2024-11-24T20:47:42,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481322585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:42,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:42,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481322585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:42,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:42,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:42,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:42,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:42,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:42,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481322585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:42,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:42,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:42,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:42,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/66fe8e73b5ba47728c819d0d728bd2bb 2024-11-24T20:47:42,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/0b4d31beebcb4e23aa2ecee9582bed3d as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/0b4d31beebcb4e23aa2ecee9582bed3d 2024-11-24T20:47:42,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/0b4d31beebcb4e23aa2ecee9582bed3d, entries=250, sequenceid=135, filesize=16.5 K 2024-11-24T20:47:42,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/8e38a712026d452195a06ffe85872191 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/8e38a712026d452195a06ffe85872191 2024-11-24T20:47:42,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/8e38a712026d452195a06ffe85872191, entries=150, sequenceid=135, filesize=11.8 K 2024-11-24T20:47:42,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/66fe8e73b5ba47728c819d0d728bd2bb as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/66fe8e73b5ba47728c819d0d728bd2bb 2024-11-24T20:47:42,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/66fe8e73b5ba47728c819d0d728bd2bb, entries=150, sequenceid=135, filesize=11.8 K 2024-11-24T20:47:42,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 1bb3e8397087a4722761784332990ead in 1279ms, sequenceid=135, compaction requested=true 2024-11-24T20:47:42,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:42,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1bb3e8397087a4722761784332990ead:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:47:42,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:42,676 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:47:42,676 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:47:42,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1bb3e8397087a4722761784332990ead:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:47:42,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:42,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1bb3e8397087a4722761784332990ead:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:47:42,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:47:42,677 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41089 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:47:42,678 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 1bb3e8397087a4722761784332990ead/A is initiating minor compaction (all files) 2024-11-24T20:47:42,678 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:47:42,678 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1bb3e8397087a4722761784332990ead/A in TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:42,678 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 1bb3e8397087a4722761784332990ead/B is initiating minor compaction (all files) 2024-11-24T20:47:42,678 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1bb3e8397087a4722761784332990ead/B in TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:42,678 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/2c3f228644ff498babadbd0491d67800, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/72bd01b05e4347f8866dfaa28dd3d3c7, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/0b4d31beebcb4e23aa2ecee9582bed3d] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp, totalSize=40.1 K 2024-11-24T20:47:42,678 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/c9c74ce5ad194289bf4775f1d708e99a, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/03d8240044ab46e5be9ac7af8a985692, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/8e38a712026d452195a06ffe85872191] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp, totalSize=35.5 K 2024-11-24T20:47:42,678 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting c9c74ce5ad194289bf4775f1d708e99a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732481257937 2024-11-24T20:47:42,679 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 03d8240044ab46e5be9ac7af8a985692, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732481258597 2024-11-24T20:47:42,679 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e38a712026d452195a06ffe85872191, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732481260760 2024-11-24T20:47:42,679 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c3f228644ff498babadbd0491d67800, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732481257937 2024-11-24T20:47:42,679 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72bd01b05e4347f8866dfaa28dd3d3c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732481258597 2024-11-24T20:47:42,680 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b4d31beebcb4e23aa2ecee9582bed3d, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732481260760 2024-11-24T20:47:42,700 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1bb3e8397087a4722761784332990ead#A#compaction#194 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:42,700 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/9668898378cc491f8644a20a9d4cee7b is 50, key is test_row_0/A:col10/1732481261396/Put/seqid=0 2024-11-24T20:47:42,706 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1bb3e8397087a4722761784332990ead#B#compaction#195 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:42,707 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/c061b759469e491baf8ae5c73965265f is 50, key is test_row_0/B:col10/1732481261396/Put/seqid=0 2024-11-24T20:47:42,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742058_1234 (size=12409) 2024-11-24T20:47:42,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742059_1235 (size=12409) 2024-11-24T20:47:42,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 1bb3e8397087a4722761784332990ead 2024-11-24T20:47:42,788 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:47:42,788 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1bb3e8397087a4722761784332990ead 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-24T20:47:42,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/97989a481d084ed2bc0c72d6cd381182 is 50, key is test_row_0/A:col10/1732481261413/Put/seqid=0 2024-11-24T20:47:42,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742060_1236 (size=14541) 2024-11-24T20:47:42,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/97989a481d084ed2bc0c72d6cd381182 2024-11-24T20:47:42,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/1537e62c871a4bb09e985e7f9709d373 is 50, key is test_row_0/B:col10/1732481261413/Put/seqid=0 2024-11-24T20:47:42,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742061_1237 (size=12151) 2024-11-24T20:47:42,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-24T20:47:42,835 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:47:42,835 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:47:42,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-24T20:47:42,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T20:47:42,837 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:47:42,838 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:47:42,838 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:47:42,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:42,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481322851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:42,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:42,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:42,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:42,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T20:47:42,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:42,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481322965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:42,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:42,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:42,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:42,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T20:47:42,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:42,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:42,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:42,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:42,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:42,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:43,116 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/9668898378cc491f8644a20a9d4cee7b as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/9668898378cc491f8644a20a9d4cee7b 2024-11-24T20:47:43,118 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/c061b759469e491baf8ae5c73965265f as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/c061b759469e491baf8ae5c73965265f 2024-11-24T20:47:43,123 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1bb3e8397087a4722761784332990ead/A of 1bb3e8397087a4722761784332990ead into 9668898378cc491f8644a20a9d4cee7b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:43,123 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:43,123 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., storeName=1bb3e8397087a4722761784332990ead/A, priority=13, startTime=1732481262676; duration=0sec 2024-11-24T20:47:43,123 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:47:43,123 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1bb3e8397087a4722761784332990ead:A 2024-11-24T20:47:43,123 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:47:43,124 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1bb3e8397087a4722761784332990ead/B of 1bb3e8397087a4722761784332990ead into c061b759469e491baf8ae5c73965265f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:43,125 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:43,125 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., storeName=1bb3e8397087a4722761784332990ead/B, priority=13, startTime=1732481262676; duration=0sec 2024-11-24T20:47:43,125 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:43,125 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1bb3e8397087a4722761784332990ead:B 2024-11-24T20:47:43,125 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:47:43,125 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 1bb3e8397087a4722761784332990ead/C is initiating minor compaction (all files) 2024-11-24T20:47:43,125 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1bb3e8397087a4722761784332990ead/C in TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:43,125 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/bc5433afd90c49d6b8219c9fa159d0e6, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/4d70fd7745614ae485465c8f672616b5, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/66fe8e73b5ba47728c819d0d728bd2bb] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp, totalSize=35.5 K 2024-11-24T20:47:43,126 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc5433afd90c49d6b8219c9fa159d0e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732481257937 2024-11-24T20:47:43,126 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d70fd7745614ae485465c8f672616b5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732481258597 2024-11-24T20:47:43,126 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66fe8e73b5ba47728c819d0d728bd2bb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732481260760 2024-11-24T20:47:43,136 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1bb3e8397087a4722761784332990ead#C#compaction#198 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:43,136 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/7d0aec188b9842119d50de7d11bcc755 is 50, key is test_row_0/C:col10/1732481261396/Put/seqid=0 2024-11-24T20:47:43,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742062_1238 (size=12409) 2024-11-24T20:47:43,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T20:47:43,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:43,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:43,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:43,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:43,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:43,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:43,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T20:47:43,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:43,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481323175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:43,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:43,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:43,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:43,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/1537e62c871a4bb09e985e7f9709d373 2024-11-24T20:47:43,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/eda703877aa44f3f843f16593d2d20c1 is 50, key is test_row_0/C:col10/1732481261413/Put/seqid=0 2024-11-24T20:47:43,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742063_1239 (size=12151) 2024-11-24T20:47:43,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T20:47:43,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:43,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:43,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:43,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:43,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:43,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:43,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T20:47:43,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:43,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:43,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:43,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:43,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:43,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:43,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T20:47:43,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:43,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481323485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:43,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:43,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:43,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:43,550 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/7d0aec188b9842119d50de7d11bcc755 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/7d0aec188b9842119d50de7d11bcc755 2024-11-24T20:47:43,559 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1bb3e8397087a4722761784332990ead/C of 1bb3e8397087a4722761784332990ead into 7d0aec188b9842119d50de7d11bcc755(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:43,560 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:43,560 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., storeName=1bb3e8397087a4722761784332990ead/C, priority=13, startTime=1732481262677; duration=0sec 2024-11-24T20:47:43,560 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:43,560 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1bb3e8397087a4722761784332990ead:C 2024-11-24T20:47:43,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:43,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481323595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:43,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:43,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:43,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481323595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:43,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481323595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:43,598 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_2' in 1bb3e8397087a4722761784332990ead of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:47:43,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:43,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:43,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:43,600 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_2' in 1bb3e8397087a4722761784332990ead of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:47:43,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:43,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:43,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:43,601 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_1' in 1bb3e8397087a4722761784332990ead of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:47:43,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:43,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:43,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:43,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T20:47:43,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:43,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:43,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:43,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:43,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:43,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:43,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/eda703877aa44f3f843f16593d2d20c1 2024-11-24T20:47:43,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:43,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1732481323666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:43,667 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_1' in 1bb3e8397087a4722761784332990ead of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:47:43,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:43,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:43,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:43,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/97989a481d084ed2bc0c72d6cd381182 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/97989a481d084ed2bc0c72d6cd381182 2024-11-24T20:47:43,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/97989a481d084ed2bc0c72d6cd381182, entries=200, sequenceid=155, filesize=14.2 K 2024-11-24T20:47:43,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/1537e62c871a4bb09e985e7f9709d373 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/1537e62c871a4bb09e985e7f9709d373 2024-11-24T20:47:43,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/1537e62c871a4bb09e985e7f9709d373, entries=150, sequenceid=155, filesize=11.9 K 2024-11-24T20:47:43,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/eda703877aa44f3f843f16593d2d20c1 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/eda703877aa44f3f843f16593d2d20c1 2024-11-24T20:47:43,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/eda703877aa44f3f843f16593d2d20c1, entries=150, sequenceid=155, filesize=11.9 K 2024-11-24T20:47:43,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 1bb3e8397087a4722761784332990ead in 915ms, sequenceid=155, compaction requested=false 2024-11-24T20:47:43,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:43,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T20:47:43,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:43,760 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 1bb3e8397087a4722761784332990ead 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-24T20:47:43,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/42f94f7f00da45bba089bcdcabffd03c is 50, key is test_row_0/A:col10/1732481262848/Put/seqid=0 2024-11-24T20:47:43,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742064_1240 (size=12151) 2024-11-24T20:47:43,830 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/42f94f7f00da45bba089bcdcabffd03c 2024-11-24T20:47:43,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/d50706098f0f4528a220e25a0bc32463 is 50, key is test_row_0/B:col10/1732481262848/Put/seqid=0 2024-11-24T20:47:43,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742065_1241 (size=12151) 2024-11-24T20:47:43,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T20:47:43,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 1bb3e8397087a4722761784332990ead 2024-11-24T20:47:43,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:44,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:44,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481324081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:44,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:44,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:44,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:44,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:44,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481324195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:44,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:44,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:44,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:44,290 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/d50706098f0f4528a220e25a0bc32463 2024-11-24T20:47:44,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/7b258327786a4077a5c7d411e2266a99 is 50, key is test_row_0/C:col10/1732481262848/Put/seqid=0 2024-11-24T20:47:44,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742066_1242 (size=12151) 2024-11-24T20:47:44,342 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/7b258327786a4077a5c7d411e2266a99 2024-11-24T20:47:44,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/42f94f7f00da45bba089bcdcabffd03c as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/42f94f7f00da45bba089bcdcabffd03c 2024-11-24T20:47:44,367 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/42f94f7f00da45bba089bcdcabffd03c, entries=150, sequenceid=174, filesize=11.9 K 2024-11-24T20:47:44,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/d50706098f0f4528a220e25a0bc32463 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/d50706098f0f4528a220e25a0bc32463 2024-11-24T20:47:44,373 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/d50706098f0f4528a220e25a0bc32463, entries=150, sequenceid=174, filesize=11.9 K 2024-11-24T20:47:44,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/7b258327786a4077a5c7d411e2266a99 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/7b258327786a4077a5c7d411e2266a99 2024-11-24T20:47:44,380 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/7b258327786a4077a5c7d411e2266a99, entries=150, sequenceid=174, filesize=11.9 K 2024-11-24T20:47:44,381 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 1bb3e8397087a4722761784332990ead in 621ms, sequenceid=174, compaction requested=true 2024-11-24T20:47:44,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:44,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:44,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-24T20:47:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-24T20:47:44,384 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-24T20:47:44,384 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5440 sec 2024-11-24T20:47:44,386 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.5490 sec 2024-11-24T20:47:44,409 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:47:44,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1bb3e8397087a4722761784332990ead 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-24T20:47:44,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 1bb3e8397087a4722761784332990ead 2024-11-24T20:47:44,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/e9f9b4208122436caa689ec15af57f89 is 50, key is test_row_0/A:col10/1732481264066/Put/seqid=0 2024-11-24T20:47:44,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742067_1243 (size=14541) 2024-11-24T20:47:44,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:44,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481324468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:44,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:44,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:44,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:44,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:44,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481324575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:44,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:44,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:44,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:44,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:44,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481324785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:44,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:44,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:44,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:44,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/e9f9b4208122436caa689ec15af57f89 2024-11-24T20:47:44,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/517492a1a91d4576add1ac362acf5327 is 50, key is test_row_0/B:col10/1732481264066/Put/seqid=0 2024-11-24T20:47:44,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742068_1244 (size=12151) 2024-11-24T20:47:44,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/517492a1a91d4576add1ac362acf5327 2024-11-24T20:47:44,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/727573c4159547f0aaabaeae9b0e4db0 is 50, key is test_row_0/C:col10/1732481264066/Put/seqid=0 2024-11-24T20:47:44,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742069_1245 (size=12151) 2024-11-24T20:47:44,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T20:47:44,985 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:47:44,986 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:47:44,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-24T20:47:44,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T20:47:44,989 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:47:44,991 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:47:44,991 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:47:45,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T20:47:45,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:45,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481325095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:45,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:45,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:45,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:45,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-24T20:47:45,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:45,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:45,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:45,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:45,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:45,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:45,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-24T20:47:45,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:45,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:45,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:45,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:45,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:45,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:45,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T20:47:45,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/727573c4159547f0aaabaeae9b0e4db0 2024-11-24T20:47:45,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/e9f9b4208122436caa689ec15af57f89 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/e9f9b4208122436caa689ec15af57f89 2024-11-24T20:47:45,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/e9f9b4208122436caa689ec15af57f89, entries=200, sequenceid=195, filesize=14.2 K 2024-11-24T20:47:45,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/517492a1a91d4576add1ac362acf5327 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/517492a1a91d4576add1ac362acf5327 2024-11-24T20:47:45,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/517492a1a91d4576add1ac362acf5327, entries=150, sequenceid=195, filesize=11.9 K 2024-11-24T20:47:45,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/727573c4159547f0aaabaeae9b0e4db0 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/727573c4159547f0aaabaeae9b0e4db0 2024-11-24T20:47:45,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/727573c4159547f0aaabaeae9b0e4db0, entries=150, sequenceid=195, filesize=11.9 K 2024-11-24T20:47:45,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 1bb3e8397087a4722761784332990ead in 969ms, sequenceid=195, compaction requested=true 2024-11-24T20:47:45,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:45,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1bb3e8397087a4722761784332990ead:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:47:45,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:45,379 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:47:45,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1bb3e8397087a4722761784332990ead:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:47:45,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:45,379 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:47:45,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1bb3e8397087a4722761784332990ead:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:47:45,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:47:45,380 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:47:45,381 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 1bb3e8397087a4722761784332990ead/B is initiating minor compaction (all files) 2024-11-24T20:47:45,381 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1bb3e8397087a4722761784332990ead/B in TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:45,381 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/c061b759469e491baf8ae5c73965265f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/1537e62c871a4bb09e985e7f9709d373, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/d50706098f0f4528a220e25a0bc32463, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/517492a1a91d4576add1ac362acf5327] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp, totalSize=47.7 K 2024-11-24T20:47:45,381 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53642 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:47:45,381 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 1bb3e8397087a4722761784332990ead/A is initiating minor compaction (all files) 2024-11-24T20:47:45,381 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1bb3e8397087a4722761784332990ead/A in TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:45,381 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/9668898378cc491f8644a20a9d4cee7b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/97989a481d084ed2bc0c72d6cd381182, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/42f94f7f00da45bba089bcdcabffd03c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/e9f9b4208122436caa689ec15af57f89] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp, totalSize=52.4 K 2024-11-24T20:47:45,382 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9668898378cc491f8644a20a9d4cee7b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732481260760 2024-11-24T20:47:45,382 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting c061b759469e491baf8ae5c73965265f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732481260760 2024-11-24T20:47:45,383 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97989a481d084ed2bc0c72d6cd381182, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732481261413 2024-11-24T20:47:45,383 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 1537e62c871a4bb09e985e7f9709d373, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732481261413 2024-11-24T20:47:45,383 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42f94f7f00da45bba089bcdcabffd03c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732481262842 2024-11-24T20:47:45,384 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting d50706098f0f4528a220e25a0bc32463, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732481262842 2024-11-24T20:47:45,384 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9f9b4208122436caa689ec15af57f89, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732481264060 2024-11-24T20:47:45,385 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 517492a1a91d4576add1ac362acf5327, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732481264060 2024-11-24T20:47:45,401 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1bb3e8397087a4722761784332990ead#A#compaction#206 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:45,401 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/60fc230f575c4b619dac0a3c90530768 is 50, key is test_row_0/A:col10/1732481264066/Put/seqid=0 2024-11-24T20:47:45,405 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1bb3e8397087a4722761784332990ead#B#compaction#207 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:45,405 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/731479cf9a7946aba59bf766f3d2e2a2 is 50, key is test_row_0/B:col10/1732481264066/Put/seqid=0 2024-11-24T20:47:45,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742070_1246 (size=12595) 2024-11-24T20:47:45,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742071_1247 (size=12595) 2024-11-24T20:47:45,451 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/731479cf9a7946aba59bf766f3d2e2a2 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/731479cf9a7946aba59bf766f3d2e2a2 2024-11-24T20:47:45,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-24T20:47:45,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:45,452 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 1bb3e8397087a4722761784332990ead 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-24T20:47:45,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/d5eb5a6130b446199761faa52a3af359 is 50, key is test_row_0/A:col10/1732481264414/Put/seqid=0 2024-11-24T20:47:45,458 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1bb3e8397087a4722761784332990ead/B of 1bb3e8397087a4722761784332990ead into 731479cf9a7946aba59bf766f3d2e2a2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:45,458 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:45,458 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., storeName=1bb3e8397087a4722761784332990ead/B, priority=12, startTime=1732481265379; duration=0sec 2024-11-24T20:47:45,458 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:47:45,458 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1bb3e8397087a4722761784332990ead:B 2024-11-24T20:47:45,458 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:47:45,460 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:47:45,460 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 1bb3e8397087a4722761784332990ead/C is initiating minor compaction (all files) 2024-11-24T20:47:45,460 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1bb3e8397087a4722761784332990ead/C in TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:45,460 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/7d0aec188b9842119d50de7d11bcc755, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/eda703877aa44f3f843f16593d2d20c1, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/7b258327786a4077a5c7d411e2266a99, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/727573c4159547f0aaabaeae9b0e4db0] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp, totalSize=47.7 K 2024-11-24T20:47:45,461 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d0aec188b9842119d50de7d11bcc755, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732481260760 2024-11-24T20:47:45,461 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting eda703877aa44f3f843f16593d2d20c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732481261413 2024-11-24T20:47:45,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742072_1248 (size=12151) 2024-11-24T20:47:45,462 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b258327786a4077a5c7d411e2266a99, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732481262842 2024-11-24T20:47:45,462 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/d5eb5a6130b446199761faa52a3af359 2024-11-24T20:47:45,463 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 727573c4159547f0aaabaeae9b0e4db0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732481264060 2024-11-24T20:47:45,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/62fcedcc075e4b228033d7f648918e2c is 50, key is test_row_0/B:col10/1732481264414/Put/seqid=0 2024-11-24T20:47:45,476 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1bb3e8397087a4722761784332990ead#C#compaction#210 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:45,477 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/4b76c9f7955a4e7382b3d5a866ce7e62 is 50, key is test_row_0/C:col10/1732481264066/Put/seqid=0 2024-11-24T20:47:45,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742073_1249 (size=12151) 2024-11-24T20:47:45,511 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/62fcedcc075e4b228033d7f648918e2c 2024-11-24T20:47:45,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/67d9e3c4738e4cd099638408deaf6008 is 50, key is test_row_0/C:col10/1732481264414/Put/seqid=0 2024-11-24T20:47:45,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742074_1250 (size=12595) 2024-11-24T20:47:45,534 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/4b76c9f7955a4e7382b3d5a866ce7e62 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/4b76c9f7955a4e7382b3d5a866ce7e62 2024-11-24T20:47:45,538 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1bb3e8397087a4722761784332990ead/C of 1bb3e8397087a4722761784332990ead into 4b76c9f7955a4e7382b3d5a866ce7e62(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:45,538 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:45,538 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., storeName=1bb3e8397087a4722761784332990ead/C, priority=12, startTime=1732481265379; duration=0sec 2024-11-24T20:47:45,539 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:45,539 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1bb3e8397087a4722761784332990ead:C 2024-11-24T20:47:45,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742075_1251 (size=12151) 2024-11-24T20:47:45,542 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/67d9e3c4738e4cd099638408deaf6008 2024-11-24T20:47:45,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/d5eb5a6130b446199761faa52a3af359 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/d5eb5a6130b446199761faa52a3af359 2024-11-24T20:47:45,551 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/d5eb5a6130b446199761faa52a3af359, entries=150, sequenceid=210, filesize=11.9 K 2024-11-24T20:47:45,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/62fcedcc075e4b228033d7f648918e2c as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/62fcedcc075e4b228033d7f648918e2c 2024-11-24T20:47:45,557 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/62fcedcc075e4b228033d7f648918e2c, entries=150, sequenceid=210, filesize=11.9 K 2024-11-24T20:47:45,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/67d9e3c4738e4cd099638408deaf6008 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/67d9e3c4738e4cd099638408deaf6008 2024-11-24T20:47:45,564 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/67d9e3c4738e4cd099638408deaf6008, entries=150, sequenceid=210, filesize=11.9 K 2024-11-24T20:47:45,565 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=0 B/0 for 1bb3e8397087a4722761784332990ead in 113ms, sequenceid=210, compaction requested=false 2024-11-24T20:47:45,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:45,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:45,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-24T20:47:45,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-24T20:47:45,569 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-24T20:47:45,569 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 576 msec 2024-11-24T20:47:45,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 583 msec 2024-11-24T20:47:45,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T20:47:45,615 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:47:45,616 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:47:45,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-24T20:47:45,617 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:47:45,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T20:47:45,618 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:47:45,618 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:47:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 1bb3e8397087a4722761784332990ead 2024-11-24T20:47:45,622 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:47:45,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1bb3e8397087a4722761784332990ead 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T20:47:45,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/b538838a0c8a4ebc94d0c6a53edeb501 is 50, key is test_row_0/A:col10/1732481265621/Put/seqid=0 2024-11-24T20:47:45,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742076_1252 (size=14541) 2024-11-24T20:47:45,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:45,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481325653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:45,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:45,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:45,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:45,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:45,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481325658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:45,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:45,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:45,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:45,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:45,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:45,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481325658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:45,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481325659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:45,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:45,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:45,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:45,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:45,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:45,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:45,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T20:47:45,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:45,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481325765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:45,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-24T20:47:45,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:45,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:45,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:45,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481325765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:45,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:45,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:45,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:45,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:45,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:45,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:45,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:45,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:45,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:45,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:45,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:45,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481325775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:45,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:45,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:45,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:45,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:45,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481325775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:45,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:45,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:45,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:45,841 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/60fc230f575c4b619dac0a3c90530768 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/60fc230f575c4b619dac0a3c90530768 2024-11-24T20:47:45,845 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1bb3e8397087a4722761784332990ead/A of 1bb3e8397087a4722761784332990ead into 60fc230f575c4b619dac0a3c90530768(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:45,845 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:45,846 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., storeName=1bb3e8397087a4722761784332990ead/A, priority=12, startTime=1732481265378; duration=0sec 2024-11-24T20:47:45,846 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:45,846 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1bb3e8397087a4722761784332990ead:A 2024-11-24T20:47:45,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-24T20:47:45,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:45,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:45,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:45,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:45,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:45,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:45,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T20:47:45,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:45,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481325975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:45,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:45,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481325975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:45,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:45,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481325985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:45,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:45,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:45,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:45,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481325985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:45,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:45,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:45,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:45,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:46,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/b538838a0c8a4ebc94d0c6a53edeb501 2024-11-24T20:47:46,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/b5451853645e4de58303d7f49a0ab2f4 is 50, key is test_row_0/B:col10/1732481265621/Put/seqid=0 2024-11-24T20:47:46,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742077_1253 (size=12151) 2024-11-24T20:47:46,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/b5451853645e4de58303d7f49a0ab2f4 2024-11-24T20:47:46,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/026fa4275ec44e549501eb5f77fda006 is 50, key is test_row_0/C:col10/1732481265621/Put/seqid=0 2024-11-24T20:47:46,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742078_1254 (size=12151) 2024-11-24T20:47:46,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-24T20:47:46,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:46,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:46,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:46,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:46,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:46,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:46,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-24T20:47:46,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:46,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:46,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:46,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:46,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:46,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:46,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T20:47:46,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:46,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481326285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:46,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:46,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481326285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:46,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:46,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:46,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:46,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:46,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:46,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:46,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:46,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481326295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:46,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:46,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481326295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:46,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:46,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:46,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:46,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:46,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:46,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:46,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-24T20:47:46,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:46,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:46,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:46,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:46,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:46,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:46,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/026fa4275ec44e549501eb5f77fda006 2024-11-24T20:47:46,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/b538838a0c8a4ebc94d0c6a53edeb501 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/b538838a0c8a4ebc94d0c6a53edeb501 2024-11-24T20:47:46,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/b538838a0c8a4ebc94d0c6a53edeb501, entries=200, sequenceid=223, filesize=14.2 K 2024-11-24T20:47:46,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/b5451853645e4de58303d7f49a0ab2f4 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/b5451853645e4de58303d7f49a0ab2f4 2024-11-24T20:47:46,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/b5451853645e4de58303d7f49a0ab2f4, entries=150, sequenceid=223, filesize=11.9 K 2024-11-24T20:47:46,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/026fa4275ec44e549501eb5f77fda006 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/026fa4275ec44e549501eb5f77fda006 2024-11-24T20:47:46,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/026fa4275ec44e549501eb5f77fda006, entries=150, sequenceid=223, filesize=11.9 K 2024-11-24T20:47:46,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1bb3e8397087a4722761784332990ead in 855ms, sequenceid=223, compaction requested=true 2024-11-24T20:47:46,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:46,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1bb3e8397087a4722761784332990ead:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:47:46,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:46,477 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:47:46,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1bb3e8397087a4722761784332990ead:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:47:46,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:46,477 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:47:46,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1bb3e8397087a4722761784332990ead:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:47:46,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:47:46,478 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:47:46,478 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:47:46,478 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 1bb3e8397087a4722761784332990ead/A is initiating minor compaction (all files) 2024-11-24T20:47:46,478 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 1bb3e8397087a4722761784332990ead/B is initiating minor compaction (all files) 2024-11-24T20:47:46,478 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1bb3e8397087a4722761784332990ead/A in TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:46,478 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1bb3e8397087a4722761784332990ead/B in TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:46,478 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/60fc230f575c4b619dac0a3c90530768, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/d5eb5a6130b446199761faa52a3af359, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/b538838a0c8a4ebc94d0c6a53edeb501] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp, totalSize=38.4 K 2024-11-24T20:47:46,479 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/731479cf9a7946aba59bf766f3d2e2a2, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/62fcedcc075e4b228033d7f648918e2c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/b5451853645e4de58303d7f49a0ab2f4] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp, totalSize=36.0 K 2024-11-24T20:47:46,479 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60fc230f575c4b619dac0a3c90530768, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732481264060 2024-11-24T20:47:46,479 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 731479cf9a7946aba59bf766f3d2e2a2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732481264060 2024-11-24T20:47:46,479 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5eb5a6130b446199761faa52a3af359, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732481264414 2024-11-24T20:47:46,479 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 62fcedcc075e4b228033d7f648918e2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732481264414 2024-11-24T20:47:46,480 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting b538838a0c8a4ebc94d0c6a53edeb501, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732481265610 2024-11-24T20:47:46,480 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting b5451853645e4de58303d7f49a0ab2f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732481265610 2024-11-24T20:47:46,488 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1bb3e8397087a4722761784332990ead#B#compaction#215 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:46,489 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/94aa7b6338e44f989603341dc43bc466 is 50, key is test_row_0/B:col10/1732481265621/Put/seqid=0 2024-11-24T20:47:46,491 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1bb3e8397087a4722761784332990ead#A#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:46,492 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/97bec558ad1f460f8d7f924d54ecc275 is 50, key is test_row_0/A:col10/1732481265621/Put/seqid=0 2024-11-24T20:47:46,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742080_1256 (size=12697) 2024-11-24T20:47:46,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742079_1255 (size=12697) 2024-11-24T20:47:46,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-24T20:47:46,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:46,535 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 1bb3e8397087a4722761784332990ead 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-24T20:47:46,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/261813ba9dba45ef8a0f0517afe4f1f3 is 50, key is test_row_0/A:col10/1732481265658/Put/seqid=0 2024-11-24T20:47:46,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742081_1257 (size=12151) 2024-11-24T20:47:46,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T20:47:46,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 1bb3e8397087a4722761784332990ead 2024-11-24T20:47:46,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:46,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:46,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481326807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:46,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:46,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481326807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:46,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:46,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:46,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:46,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:46,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481326808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:46,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:46,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:46,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:46,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:46,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:46,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:46,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:46,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481326809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:46,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:46,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:46,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:46,915 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/94aa7b6338e44f989603341dc43bc466 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/94aa7b6338e44f989603341dc43bc466 2024-11-24T20:47:46,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:46,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481326916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:46,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:46,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:46,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:46,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481326916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:46,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:46,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:46,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:46,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:46,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:46,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481326916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:46,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:46,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:46,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:46,921 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1bb3e8397087a4722761784332990ead/B of 1bb3e8397087a4722761784332990ead into 94aa7b6338e44f989603341dc43bc466(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:46,921 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/97bec558ad1f460f8d7f924d54ecc275 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/97bec558ad1f460f8d7f924d54ecc275 2024-11-24T20:47:46,921 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:46,921 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., storeName=1bb3e8397087a4722761784332990ead/B, priority=13, startTime=1732481266477; duration=0sec 2024-11-24T20:47:46,921 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:47:46,921 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1bb3e8397087a4722761784332990ead:B 2024-11-24T20:47:46,922 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:47:46,922 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:47:46,923 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 1bb3e8397087a4722761784332990ead/C is initiating minor compaction (all files) 2024-11-24T20:47:46,923 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1bb3e8397087a4722761784332990ead/C in TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:46,923 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/4b76c9f7955a4e7382b3d5a866ce7e62, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/67d9e3c4738e4cd099638408deaf6008, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/026fa4275ec44e549501eb5f77fda006] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp, totalSize=36.0 K 2024-11-24T20:47:46,923 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b76c9f7955a4e7382b3d5a866ce7e62, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732481264060 2024-11-24T20:47:46,924 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 67d9e3c4738e4cd099638408deaf6008, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732481264414 2024-11-24T20:47:46,924 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 026fa4275ec44e549501eb5f77fda006, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732481265610 2024-11-24T20:47:46,926 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1bb3e8397087a4722761784332990ead/A of 1bb3e8397087a4722761784332990ead into 97bec558ad1f460f8d7f924d54ecc275(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:46,926 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:46,927 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., storeName=1bb3e8397087a4722761784332990ead/A, priority=13, startTime=1732481266477; duration=0sec 2024-11-24T20:47:46,927 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:46,927 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1bb3e8397087a4722761784332990ead:A 2024-11-24T20:47:46,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:46,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481326925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:46,937 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1bb3e8397087a4722761784332990ead#C#compaction#218 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:46,938 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/b0f95e10e8834d3a8d98cce743e6866f is 50, key is test_row_0/C:col10/1732481265621/Put/seqid=0 2024-11-24T20:47:46,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742082_1258 (size=12697) 2024-11-24T20:47:46,956 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/261813ba9dba45ef8a0f0517afe4f1f3 2024-11-24T20:47:46,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/b8fb0cb364bc4dfebcf0ef07b627912f is 50, key is test_row_0/B:col10/1732481265658/Put/seqid=0 2024-11-24T20:47:46,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742083_1259 (size=12151) 2024-11-24T20:47:47,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:47,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481327125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:47,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:47,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:47,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481327125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:47,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:47,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:47,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:47,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:47,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:47,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:47,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481327126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:47,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:47,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:47,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:47,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:47,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481327135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:47,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:47,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:47,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:47,347 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/b0f95e10e8834d3a8d98cce743e6866f as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/b0f95e10e8834d3a8d98cce743e6866f 2024-11-24T20:47:47,352 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1bb3e8397087a4722761784332990ead/C of 1bb3e8397087a4722761784332990ead into b0f95e10e8834d3a8d98cce743e6866f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:47,352 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:47,352 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., storeName=1bb3e8397087a4722761784332990ead/C, priority=13, startTime=1732481266477; duration=0sec 2024-11-24T20:47:47,352 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:47,352 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1bb3e8397087a4722761784332990ead:C 2024-11-24T20:47:47,370 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/b8fb0cb364bc4dfebcf0ef07b627912f 2024-11-24T20:47:47,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/f2c8c55cd9d347db9ec63c35d73a1e9b is 50, key is test_row_0/C:col10/1732481265658/Put/seqid=0 2024-11-24T20:47:47,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742084_1260 (size=12151) 2024-11-24T20:47:47,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:47,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481327435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:47,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:47,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:47,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:47,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:47,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481327435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:47,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:47,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:47,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:47,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:47,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481327435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:47,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:47,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:47,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:47,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:47,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481327444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:47,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:47,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:47,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:47,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T20:47:47,782 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/f2c8c55cd9d347db9ec63c35d73a1e9b 2024-11-24T20:47:47,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/261813ba9dba45ef8a0f0517afe4f1f3 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/261813ba9dba45ef8a0f0517afe4f1f3 2024-11-24T20:47:47,790 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/261813ba9dba45ef8a0f0517afe4f1f3, entries=150, sequenceid=249, filesize=11.9 K 2024-11-24T20:47:47,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/b8fb0cb364bc4dfebcf0ef07b627912f as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/b8fb0cb364bc4dfebcf0ef07b627912f 2024-11-24T20:47:47,795 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/b8fb0cb364bc4dfebcf0ef07b627912f, entries=150, sequenceid=249, filesize=11.9 K 2024-11-24T20:47:47,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/f2c8c55cd9d347db9ec63c35d73a1e9b as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/f2c8c55cd9d347db9ec63c35d73a1e9b 2024-11-24T20:47:47,800 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/f2c8c55cd9d347db9ec63c35d73a1e9b, entries=150, sequenceid=249, filesize=11.9 K 2024-11-24T20:47:47,801 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 1bb3e8397087a4722761784332990ead in 1266ms, sequenceid=249, compaction requested=false 2024-11-24T20:47:47,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:47,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:47,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-24T20:47:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-24T20:47:47,804 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-24T20:47:47,804 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1840 sec 2024-11-24T20:47:47,805 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.1880 sec 2024-11-24T20:47:47,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 1bb3e8397087a4722761784332990ead 2024-11-24T20:47:47,947 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:47:47,947 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1bb3e8397087a4722761784332990ead 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-24T20:47:47,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/a656707f14d84e39ac5db00b5c79a469 is 50, key is test_row_0/A:col10/1732481266806/Put/seqid=0 2024-11-24T20:47:47,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742085_1261 (size=12251) 2024-11-24T20:47:48,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:48,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481327998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:48,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:48,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:48,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:48,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481327998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:48,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:48,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:48,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481327999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:48,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:48,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481328000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:48,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:48,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:48,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:48,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:48,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:48,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:48,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:48,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:48,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:48,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:48,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481328105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:48,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:48,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:48,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:48,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:48,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481328115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:48,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:48,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:48,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481328115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:48,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:48,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:48,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:48,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:48,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:48,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:48,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481328116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:48,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:48,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:48,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:48,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:48,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481328315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:48,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:48,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:48,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:48,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481328325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:48,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:48,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481328325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:48,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:48,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481328325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:48,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/a656707f14d84e39ac5db00b5c79a469 2024-11-24T20:47:48,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/81fb258006464f2eb9b4a80ccb07b1d5 is 50, key is test_row_0/B:col10/1732481266806/Put/seqid=0 2024-11-24T20:47:48,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742086_1262 (size=12251) 2024-11-24T20:47:48,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/81fb258006464f2eb9b4a80ccb07b1d5 2024-11-24T20:47:48,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/03bb999b8cf442fda5dce3d3a614f1ce is 50, key is test_row_0/C:col10/1732481266806/Put/seqid=0 2024-11-24T20:47:48,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742087_1263 (size=12251) 2024-11-24T20:47:48,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:48,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481328625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:48,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:48,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:48,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:48,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:48,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481328634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:48,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:48,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:48,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:48,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:48,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481328635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:48,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:48,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:48,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:48,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:48,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481328635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:48,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:48,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:48,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:48,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/03bb999b8cf442fda5dce3d3a614f1ce 2024-11-24T20:47:48,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/a656707f14d84e39ac5db00b5c79a469 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/a656707f14d84e39ac5db00b5c79a469 2024-11-24T20:47:48,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/a656707f14d84e39ac5db00b5c79a469, entries=150, sequenceid=265, filesize=12.0 K 2024-11-24T20:47:48,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/81fb258006464f2eb9b4a80ccb07b1d5 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/81fb258006464f2eb9b4a80ccb07b1d5 2024-11-24T20:47:48,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/81fb258006464f2eb9b4a80ccb07b1d5, entries=150, sequenceid=265, filesize=12.0 K 2024-11-24T20:47:48,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/03bb999b8cf442fda5dce3d3a614f1ce as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/03bb999b8cf442fda5dce3d3a614f1ce 2024-11-24T20:47:48,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/03bb999b8cf442fda5dce3d3a614f1ce, entries=150, sequenceid=265, filesize=12.0 K 2024-11-24T20:47:48,803 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 1bb3e8397087a4722761784332990ead in 856ms, sequenceid=265, compaction requested=true 2024-11-24T20:47:48,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:48,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1bb3e8397087a4722761784332990ead:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:47:48,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:48,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1bb3e8397087a4722761784332990ead:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:47:48,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:48,803 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:47:48,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1bb3e8397087a4722761784332990ead:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:47:48,803 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:47:48,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:47:48,804 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:47:48,804 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:47:48,804 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 1bb3e8397087a4722761784332990ead/A is initiating minor compaction (all files) 2024-11-24T20:47:48,804 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 1bb3e8397087a4722761784332990ead/B is initiating minor compaction (all files) 2024-11-24T20:47:48,805 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1bb3e8397087a4722761784332990ead/A in TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:48,805 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1bb3e8397087a4722761784332990ead/B in TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:48,805 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/94aa7b6338e44f989603341dc43bc466, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/b8fb0cb364bc4dfebcf0ef07b627912f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/81fb258006464f2eb9b4a80ccb07b1d5] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp, totalSize=36.2 K 2024-11-24T20:47:48,805 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/97bec558ad1f460f8d7f924d54ecc275, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/261813ba9dba45ef8a0f0517afe4f1f3, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/a656707f14d84e39ac5db00b5c79a469] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp, totalSize=36.2 K 2024-11-24T20:47:48,805 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97bec558ad1f460f8d7f924d54ecc275, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732481265610 2024-11-24T20:47:48,805 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 94aa7b6338e44f989603341dc43bc466, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732481265610 2024-11-24T20:47:48,805 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting b8fb0cb364bc4dfebcf0ef07b627912f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732481265643 2024-11-24T20:47:48,805 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 261813ba9dba45ef8a0f0517afe4f1f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732481265643 2024-11-24T20:47:48,806 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 81fb258006464f2eb9b4a80ccb07b1d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732481266801 2024-11-24T20:47:48,806 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting a656707f14d84e39ac5db00b5c79a469, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732481266801 2024-11-24T20:47:48,813 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1bb3e8397087a4722761784332990ead#A#compaction#224 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:48,814 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/f60445ee0241477ab00d0c9821c19a60 is 50, key is test_row_0/A:col10/1732481266806/Put/seqid=0 2024-11-24T20:47:48,817 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1bb3e8397087a4722761784332990ead#B#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:48,817 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/f1f67e366c774e97aee2d168900621d3 is 50, key is test_row_0/B:col10/1732481266806/Put/seqid=0 2024-11-24T20:47:48,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742088_1264 (size=12899) 2024-11-24T20:47:48,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742089_1265 (size=12899) 2024-11-24T20:47:49,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 1bb3e8397087a4722761784332990ead 2024-11-24T20:47:49,137 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:47:49,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1bb3e8397087a4722761784332990ead 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-24T20:47:49,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/e2b43012d9b941e6a3a43bd6737dbe0b is 50, key is test_row_0/A:col10/1732481267997/Put/seqid=0 2024-11-24T20:47:49,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742090_1266 (size=12301) 2024-11-24T20:47:49,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:49,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:49,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481329154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:49,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481329155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:49,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:49,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:49,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:49,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481329155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:49,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:49,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:49,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481329155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:49,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:49,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:49,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:49,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:49,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:49,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:49,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:49,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:49,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:49,230 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/f1f67e366c774e97aee2d168900621d3 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/f1f67e366c774e97aee2d168900621d3 2024-11-24T20:47:49,230 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/f60445ee0241477ab00d0c9821c19a60 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/f60445ee0241477ab00d0c9821c19a60 2024-11-24T20:47:49,234 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1bb3e8397087a4722761784332990ead/A of 1bb3e8397087a4722761784332990ead into f60445ee0241477ab00d0c9821c19a60(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:49,234 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1bb3e8397087a4722761784332990ead/B of 1bb3e8397087a4722761784332990ead into f1f67e366c774e97aee2d168900621d3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:49,234 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:49,234 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:49,234 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., storeName=1bb3e8397087a4722761784332990ead/A, priority=13, startTime=1732481268803; duration=0sec 2024-11-24T20:47:49,234 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., storeName=1bb3e8397087a4722761784332990ead/B, priority=13, startTime=1732481268803; duration=0sec 2024-11-24T20:47:49,234 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:47:49,234 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1bb3e8397087a4722761784332990ead:A 2024-11-24T20:47:49,234 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:49,234 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1bb3e8397087a4722761784332990ead:B 2024-11-24T20:47:49,234 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:47:49,235 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:47:49,235 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 1bb3e8397087a4722761784332990ead/C is initiating minor compaction (all files) 2024-11-24T20:47:49,235 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1bb3e8397087a4722761784332990ead/C in TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:49,235 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/b0f95e10e8834d3a8d98cce743e6866f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/f2c8c55cd9d347db9ec63c35d73a1e9b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/03bb999b8cf442fda5dce3d3a614f1ce] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp, totalSize=36.2 K 2024-11-24T20:47:49,236 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0f95e10e8834d3a8d98cce743e6866f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732481265610 2024-11-24T20:47:49,236 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2c8c55cd9d347db9ec63c35d73a1e9b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732481265643 2024-11-24T20:47:49,236 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03bb999b8cf442fda5dce3d3a614f1ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732481266801 2024-11-24T20:47:49,241 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1bb3e8397087a4722761784332990ead#C#compaction#227 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:49,242 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/c4f5826051d1471483af4f414bd9d6f1 is 50, key is test_row_0/C:col10/1732481266806/Put/seqid=0 2024-11-24T20:47:49,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742091_1267 (size=12899) 2024-11-24T20:47:49,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:49,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481329265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:49,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:49,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481329265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:49,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:49,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481329265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:49,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:49,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:49,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:49,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:49,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:49,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481329265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:49,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:49,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:49,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:49,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:49,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481329475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:49,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:49,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:49,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481329475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:49,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481329475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:49,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:49,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:49,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481329475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:49,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:49,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:49,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:49,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:49,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:49,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:49,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:49,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:49,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:49,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:49,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:49,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/e2b43012d9b941e6a3a43bd6737dbe0b 2024-11-24T20:47:49,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/2d3adc2a958a46e39fc8ba65ea8da9f8 is 50, key is test_row_0/B:col10/1732481267997/Put/seqid=0 2024-11-24T20:47:49,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742092_1268 (size=12301) 2024-11-24T20:47:49,698 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/c4f5826051d1471483af4f414bd9d6f1 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/c4f5826051d1471483af4f414bd9d6f1 2024-11-24T20:47:49,702 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1bb3e8397087a4722761784332990ead/C of 1bb3e8397087a4722761784332990ead into c4f5826051d1471483af4f414bd9d6f1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:49,702 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:49,702 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., storeName=1bb3e8397087a4722761784332990ead/C, priority=13, startTime=1732481268803; duration=0sec 2024-11-24T20:47:49,702 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:49,702 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1bb3e8397087a4722761784332990ead:C 2024-11-24T20:47:49,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T20:47:49,786 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:47:49,786 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:47:49,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-24T20:47:49,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:49,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481329786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:49,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:49,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:49,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:49,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:49,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481329786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:49,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T20:47:49,788 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:47:49,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:49,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:49,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:49,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:49,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481329786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:49,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:49,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:49,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:49,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:49,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481329786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:49,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:49,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:49,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:49,789 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:47:49,789 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:47:49,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T20:47:49,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-24T20:47:49,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:49,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:49,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:49,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:49,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:49,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:49,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/2d3adc2a958a46e39fc8ba65ea8da9f8 2024-11-24T20:47:49,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/5cbc721bdccf4ee59d8827a7500cb75e is 50, key is test_row_0/C:col10/1732481267997/Put/seqid=0 2024-11-24T20:47:49,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742093_1269 (size=12301) 2024-11-24T20:47:50,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-24T20:47:50,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:50,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:50,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:50,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:50,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:50,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:50,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T20:47:50,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-24T20:47:50,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:50,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:50,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:50,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:50,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:50,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:50,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:50,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481330295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:50,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:50,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481330295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:50,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:50,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481330295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:50,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:50,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:50,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:50,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:50,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:50,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:50,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:50,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:50,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:50,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481330295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:50,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:50,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:50,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:50,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:50,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/5cbc721bdccf4ee59d8827a7500cb75e 2024-11-24T20:47:50,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/e2b43012d9b941e6a3a43bd6737dbe0b as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/e2b43012d9b941e6a3a43bd6737dbe0b 2024-11-24T20:47:50,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/e2b43012d9b941e6a3a43bd6737dbe0b, entries=150, sequenceid=289, filesize=12.0 K 2024-11-24T20:47:50,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/2d3adc2a958a46e39fc8ba65ea8da9f8 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/2d3adc2a958a46e39fc8ba65ea8da9f8 2024-11-24T20:47:50,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/2d3adc2a958a46e39fc8ba65ea8da9f8, entries=150, sequenceid=289, filesize=12.0 K 2024-11-24T20:47:50,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/5cbc721bdccf4ee59d8827a7500cb75e as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/5cbc721bdccf4ee59d8827a7500cb75e 2024-11-24T20:47:50,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/5cbc721bdccf4ee59d8827a7500cb75e, entries=150, sequenceid=289, filesize=12.0 K 2024-11-24T20:47:50,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 1bb3e8397087a4722761784332990ead in 1253ms, sequenceid=289, compaction requested=false 2024-11-24T20:47:50,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:50,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-24T20:47:50,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:50,400 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 1bb3e8397087a4722761784332990ead 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-24T20:47:50,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/411143cd754742b5971f0baba1a5facd is 50, key is test_row_0/A:col10/1732481269148/Put/seqid=0 2024-11-24T20:47:50,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T20:47:50,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742094_1270 (size=9857) 2024-11-24T20:47:50,433 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/411143cd754742b5971f0baba1a5facd 2024-11-24T20:47:50,443 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T20:47:50,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/3eba0f9f3d3d4fd2ae5852874878c03d is 50, key is test_row_0/B:col10/1732481269148/Put/seqid=0 2024-11-24T20:47:50,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742095_1271 (size=9857) 2024-11-24T20:47:50,467 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/3eba0f9f3d3d4fd2ae5852874878c03d 2024-11-24T20:47:50,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/ded90923c5694efebee1fa156b6ed932 is 50, key is test_row_0/C:col10/1732481269148/Put/seqid=0 2024-11-24T20:47:50,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742096_1272 (size=9857) 2024-11-24T20:47:50,506 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/ded90923c5694efebee1fa156b6ed932 2024-11-24T20:47:50,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/411143cd754742b5971f0baba1a5facd as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/411143cd754742b5971f0baba1a5facd 2024-11-24T20:47:50,517 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/411143cd754742b5971f0baba1a5facd, entries=100, sequenceid=305, filesize=9.6 K 2024-11-24T20:47:50,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/3eba0f9f3d3d4fd2ae5852874878c03d as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/3eba0f9f3d3d4fd2ae5852874878c03d 2024-11-24T20:47:50,523 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/3eba0f9f3d3d4fd2ae5852874878c03d, entries=100, sequenceid=305, filesize=9.6 K 2024-11-24T20:47:50,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/ded90923c5694efebee1fa156b6ed932 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/ded90923c5694efebee1fa156b6ed932 2024-11-24T20:47:50,532 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/ded90923c5694efebee1fa156b6ed932, entries=100, sequenceid=305, filesize=9.6 K 2024-11-24T20:47:50,534 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=0 B/0 for 1bb3e8397087a4722761784332990ead in 133ms, sequenceid=305, compaction requested=true 2024-11-24T20:47:50,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:50,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:50,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-24T20:47:50,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-24T20:47:50,537 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-24T20:47:50,537 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 746 msec 2024-11-24T20:47:50,539 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 752 msec 2024-11-24T20:47:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T20:47:50,925 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:47:50,925 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:47:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-24T20:47:50,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-24T20:47:50,931 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:47:50,932 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:47:50,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:47:51,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-24T20:47:51,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-24T20:47:51,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:51,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:51,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:51,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-24T20:47:51,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-24T20:47:51,090 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-24T20:47:51,090 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 156 msec 2024-11-24T20:47:51,093 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 166 msec 2024-11-24T20:47:51,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-24T20:47:51,245 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:47:51,246 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:47:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-24T20:47:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T20:47:51,247 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:47:51,248 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:47:51,249 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:47:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 1bb3e8397087a4722761784332990ead 2024-11-24T20:47:51,315 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:47:51,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1bb3e8397087a4722761784332990ead 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T20:47:51,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/9be698c9bd02436b8a7ad209256aada6 is 50, key is test_row_0/A:col10/1732481271314/Put/seqid=0 2024-11-24T20:47:51,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:51,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481331344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481331344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:51,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:51,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:51,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481331346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:51,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:51,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:51,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:51,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:51,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:51,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:51,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T20:47:51,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481331353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:51,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:51,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:51,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:51,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742097_1273 (size=12301) 2024-11-24T20:47:51,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-24T20:47:51,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:51,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:51,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:51,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:51,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:51,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481331455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:51,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481331455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:51,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:51,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:51,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:51,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:51,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:51,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:51,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481331455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:51,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:51,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:51,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:51,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481331465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:51,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:51,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:51,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:51,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-24T20:47:51,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:51,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:51,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:51,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:51,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T20:47:51,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481331665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:51,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:51,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:51,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:51,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481331666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:51,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481331666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:51,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:51,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:51,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:51,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:51,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:51,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:51,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481331676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:51,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:51,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:51,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:51,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-24T20:47:51,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:51,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:51,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:51,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:51,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:51,789 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/9be698c9bd02436b8a7ad209256aada6 2024-11-24T20:47:51,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/07e7cefd293a45069fce2d0a6778cb92 is 50, key is test_row_0/B:col10/1732481271314/Put/seqid=0 2024-11-24T20:47:51,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742098_1274 (size=12301) 2024-11-24T20:47:51,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/07e7cefd293a45069fce2d0a6778cb92 2024-11-24T20:47:51,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/ec7cdc394592499ba29170285a3b393c is 50, key is test_row_0/C:col10/1732481271314/Put/seqid=0 2024-11-24T20:47:51,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742099_1275 (size=12301) 2024-11-24T20:47:51,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/ec7cdc394592499ba29170285a3b393c 2024-11-24T20:47:51,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/9be698c9bd02436b8a7ad209256aada6 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/9be698c9bd02436b8a7ad209256aada6 2024-11-24T20:47:51,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-24T20:47:51,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:51,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:51,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:51,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:51,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:51,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/9be698c9bd02436b8a7ad209256aada6, entries=150, sequenceid=318, filesize=12.0 K 2024-11-24T20:47:51,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/07e7cefd293a45069fce2d0a6778cb92 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/07e7cefd293a45069fce2d0a6778cb92 2024-11-24T20:47:51,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/07e7cefd293a45069fce2d0a6778cb92, entries=150, sequenceid=318, filesize=12.0 K 2024-11-24T20:47:51,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/ec7cdc394592499ba29170285a3b393c as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/ec7cdc394592499ba29170285a3b393c 2024-11-24T20:47:51,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T20:47:51,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/ec7cdc394592499ba29170285a3b393c, entries=150, sequenceid=318, filesize=12.0 K 2024-11-24T20:47:51,879 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 1bb3e8397087a4722761784332990ead in 564ms, sequenceid=318, compaction requested=true 2024-11-24T20:47:51,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:51,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1bb3e8397087a4722761784332990ead:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:47:51,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:51,879 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:47:51,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1bb3e8397087a4722761784332990ead:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:47:51,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:51,879 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:47:51,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1bb3e8397087a4722761784332990ead:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:47:51,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:47:51,881 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47358 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:47:51,881 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47358 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:47:51,881 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 1bb3e8397087a4722761784332990ead/A is initiating minor compaction (all files) 2024-11-24T20:47:51,881 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 1bb3e8397087a4722761784332990ead/B is initiating minor compaction (all files) 2024-11-24T20:47:51,881 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1bb3e8397087a4722761784332990ead/A in TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:51,881 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1bb3e8397087a4722761784332990ead/B in TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:51,881 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/f1f67e366c774e97aee2d168900621d3, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/2d3adc2a958a46e39fc8ba65ea8da9f8, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/3eba0f9f3d3d4fd2ae5852874878c03d, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/07e7cefd293a45069fce2d0a6778cb92] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp, totalSize=46.2 K 2024-11-24T20:47:51,881 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/f60445ee0241477ab00d0c9821c19a60, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/e2b43012d9b941e6a3a43bd6737dbe0b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/411143cd754742b5971f0baba1a5facd, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/9be698c9bd02436b8a7ad209256aada6] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp, totalSize=46.2 K 2024-11-24T20:47:51,882 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting f60445ee0241477ab00d0c9821c19a60, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732481266801 2024-11-24T20:47:51,883 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting f1f67e366c774e97aee2d168900621d3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732481266801 2024-11-24T20:47:51,883 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d3adc2a958a46e39fc8ba65ea8da9f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732481267997 2024-11-24T20:47:51,883 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2b43012d9b941e6a3a43bd6737dbe0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732481267997 2024-11-24T20:47:51,883 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 411143cd754742b5971f0baba1a5facd, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732481269148 2024-11-24T20:47:51,883 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 3eba0f9f3d3d4fd2ae5852874878c03d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732481269148 2024-11-24T20:47:51,884 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 07e7cefd293a45069fce2d0a6778cb92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732481271305 2024-11-24T20:47:51,884 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9be698c9bd02436b8a7ad209256aada6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732481271305 2024-11-24T20:47:51,896 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1bb3e8397087a4722761784332990ead#A#compaction#236 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:51,897 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/fd3cd083b310435ca6db1e5c1355cd6a is 50, key is test_row_0/A:col10/1732481271314/Put/seqid=0 2024-11-24T20:47:51,898 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1bb3e8397087a4722761784332990ead#B#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:51,899 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/bd38d819b1f24afd8c75a7f5ebcb600b is 50, key is test_row_0/B:col10/1732481271314/Put/seqid=0 2024-11-24T20:47:51,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742100_1276 (size=13085) 2024-11-24T20:47:51,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742101_1277 (size=13085) 2024-11-24T20:47:51,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 1bb3e8397087a4722761784332990ead 2024-11-24T20:47:51,978 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:47:51,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1bb3e8397087a4722761784332990ead 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-24T20:47:51,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/60921cf5c8c54d25b5e0c95c6f6325ec is 50, key is test_row_0/A:col10/1732481271978/Put/seqid=0 2024-11-24T20:47:51,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742102_1278 (size=17181) 2024-11-24T20:47:52,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-24T20:47:52,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:52,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:52,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:52,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:52,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:52,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:52,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:52,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481331986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:52,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:52,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:52,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:52,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:52,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481332021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:52,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:52,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481332022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:52,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:52,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:52,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:52,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:52,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:52,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:52,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481332022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:52,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:52,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:52,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:52,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:52,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:52,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481332125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:52,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:52,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:52,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:52,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:52,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481332135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:52,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:52,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:52,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481332135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:52,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:52,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:52,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:52,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:52,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:52,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:52,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481332135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:52,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:52,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:52,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:52,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-24T20:47:52,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:52,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:52,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:52,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:52,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:52,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:52,316 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/fd3cd083b310435ca6db1e5c1355cd6a as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/fd3cd083b310435ca6db1e5c1355cd6a 2024-11-24T20:47:52,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-24T20:47:52,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:52,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:52,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:52,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:52,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:52,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:52,334 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1bb3e8397087a4722761784332990ead/A of 1bb3e8397087a4722761784332990ead into fd3cd083b310435ca6db1e5c1355cd6a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:52,334 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:52,334 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., storeName=1bb3e8397087a4722761784332990ead/A, priority=12, startTime=1732481271879; duration=0sec 2024-11-24T20:47:52,334 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:47:52,334 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1bb3e8397087a4722761784332990ead:A 2024-11-24T20:47:52,334 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:47:52,337 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47358 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:47:52,337 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 1bb3e8397087a4722761784332990ead/C is initiating minor compaction (all files) 2024-11-24T20:47:52,337 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1bb3e8397087a4722761784332990ead/C in TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:52,337 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/c4f5826051d1471483af4f414bd9d6f1, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/5cbc721bdccf4ee59d8827a7500cb75e, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/ded90923c5694efebee1fa156b6ed932, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/ec7cdc394592499ba29170285a3b393c] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp, totalSize=46.2 K 2024-11-24T20:47:52,338 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4f5826051d1471483af4f414bd9d6f1, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732481266801 2024-11-24T20:47:52,338 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/bd38d819b1f24afd8c75a7f5ebcb600b as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/bd38d819b1f24afd8c75a7f5ebcb600b 2024-11-24T20:47:52,339 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cbc721bdccf4ee59d8827a7500cb75e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732481267997 2024-11-24T20:47:52,340 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting ded90923c5694efebee1fa156b6ed932, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732481269148 2024-11-24T20:47:52,341 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec7cdc394592499ba29170285a3b393c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732481271305 2024-11-24T20:47:52,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481332335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:52,346 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1bb3e8397087a4722761784332990ead/B of 1bb3e8397087a4722761784332990ead into bd38d819b1f24afd8c75a7f5ebcb600b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:52,346 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:52,346 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., storeName=1bb3e8397087a4722761784332990ead/B, priority=12, startTime=1732481271879; duration=0sec 2024-11-24T20:47:52,346 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:52,346 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1bb3e8397087a4722761784332990ead:B 2024-11-24T20:47:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:52,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481332346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:52,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:52,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:52,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:52,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:52,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481332346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:52,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:52,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:52,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:52,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481332346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:52,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:52,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:52,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:52,358 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1bb3e8397087a4722761784332990ead#C#compaction#239 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:52,359 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/e641cb05d0ed4a2988f78e383abd6796 is 50, key is test_row_0/C:col10/1732481271314/Put/seqid=0 2024-11-24T20:47:52,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742103_1279 (size=13085) 2024-11-24T20:47:52,370 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/e641cb05d0ed4a2988f78e383abd6796 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/e641cb05d0ed4a2988f78e383abd6796 2024-11-24T20:47:52,375 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1bb3e8397087a4722761784332990ead/C of 1bb3e8397087a4722761784332990ead into e641cb05d0ed4a2988f78e383abd6796(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:52,375 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:52,375 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., storeName=1bb3e8397087a4722761784332990ead/C, priority=12, startTime=1732481271879; duration=0sec 2024-11-24T20:47:52,375 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:52,375 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1bb3e8397087a4722761784332990ead:C 2024-11-24T20:47:52,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T20:47:52,388 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/60921cf5c8c54d25b5e0c95c6f6325ec 2024-11-24T20:47:52,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/30a0aef9ae0a4910b678e5860c3d2cba is 50, key is test_row_0/B:col10/1732481271978/Put/seqid=0 2024-11-24T20:47:52,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742104_1280 (size=12301) 2024-11-24T20:47:52,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/30a0aef9ae0a4910b678e5860c3d2cba 2024-11-24T20:47:52,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/3726977901c948969bcb743d87d4624d is 50, key is test_row_0/C:col10/1732481271978/Put/seqid=0 2024-11-24T20:47:52,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742105_1281 (size=12301) 2024-11-24T20:47:52,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-24T20:47:52,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:52,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:52,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:52,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:52,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:52,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-24T20:47:52,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:52,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:52,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:52,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:52,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:52,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481332656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:52,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:52,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481332656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:52,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:52,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:52,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:52,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:52,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:52,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:52,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:52,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481332656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:52,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:52,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:52,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:52,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481332665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:52,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:52,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:52,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:52,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-24T20:47:52,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:52,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:52,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:52,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:52,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:52,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:47:52,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/3726977901c948969bcb743d87d4624d 2024-11-24T20:47:52,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/60921cf5c8c54d25b5e0c95c6f6325ec as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/60921cf5c8c54d25b5e0c95c6f6325ec 2024-11-24T20:47:52,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/60921cf5c8c54d25b5e0c95c6f6325ec, entries=250, sequenceid=346, filesize=16.8 K 2024-11-24T20:47:52,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/30a0aef9ae0a4910b678e5860c3d2cba as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/30a0aef9ae0a4910b678e5860c3d2cba 2024-11-24T20:47:52,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/30a0aef9ae0a4910b678e5860c3d2cba, entries=150, sequenceid=346, filesize=12.0 K 2024-11-24T20:47:52,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/3726977901c948969bcb743d87d4624d as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/3726977901c948969bcb743d87d4624d 2024-11-24T20:47:52,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/3726977901c948969bcb743d87d4624d, entries=150, sequenceid=346, filesize=12.0 K 2024-11-24T20:47:52,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=46.96 KB/48090 for 1bb3e8397087a4722761784332990ead in 907ms, sequenceid=346, compaction requested=false 2024-11-24T20:47:52,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:52,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-24T20:47:52,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:52,937 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 1bb3e8397087a4722761784332990ead 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-24T20:47:52,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/f58bcd7cf62d46088856beff0183db4a is 50, key is test_row_0/A:col10/1732481271988/Put/seqid=0 2024-11-24T20:47:52,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742106_1282 (size=12301) 2024-11-24T20:47:52,955 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/f58bcd7cf62d46088856beff0183db4a 2024-11-24T20:47:52,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/e888621edbd84f0ea0620b7aa4e55a0c is 50, key is test_row_0/B:col10/1732481271988/Put/seqid=0 2024-11-24T20:47:52,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742107_1283 (size=12301) 2024-11-24T20:47:53,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 1bb3e8397087a4722761784332990ead 2024-11-24T20:47:53,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. as already flushing 2024-11-24T20:47:53,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:53,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:53,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481333203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:53,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481333204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:53,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:53,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481333205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:53,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:53,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:53,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481333207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:53,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:53,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:53,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481333315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:53,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481333315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:53,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481333315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:53,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:53,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:53,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481333315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:53,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:53,379 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/e888621edbd84f0ea0620b7aa4e55a0c 2024-11-24T20:47:53,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/9a38fec053bf4271b2f22e3f1ed8f13f is 50, key is test_row_0/C:col10/1732481271988/Put/seqid=0 2024-11-24T20:47:53,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742108_1284 (size=12301) 2024-11-24T20:47:53,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T20:47:53,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:53,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481333525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:53,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:53,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481333525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:53,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:53,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481333525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:53,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:53,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481333526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:53,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:53,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:53,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:53,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:53,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:53,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42296 deadline: 1732481333755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:53,757 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_1' in 1bb3e8397087a4722761784332990ead of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18325 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:47:53,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:53,790 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/9a38fec053bf4271b2f22e3f1ed8f13f 2024-11-24T20:47:53,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/f58bcd7cf62d46088856beff0183db4a as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/f58bcd7cf62d46088856beff0183db4a 2024-11-24T20:47:53,805 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/f58bcd7cf62d46088856beff0183db4a, entries=150, sequenceid=359, filesize=12.0 K 2024-11-24T20:47:53,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/e888621edbd84f0ea0620b7aa4e55a0c as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/e888621edbd84f0ea0620b7aa4e55a0c 2024-11-24T20:47:53,810 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/e888621edbd84f0ea0620b7aa4e55a0c, entries=150, sequenceid=359, filesize=12.0 K 2024-11-24T20:47:53,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/9a38fec053bf4271b2f22e3f1ed8f13f as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/9a38fec053bf4271b2f22e3f1ed8f13f 2024-11-24T20:47:53,814 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/9a38fec053bf4271b2f22e3f1ed8f13f, entries=150, sequenceid=359, filesize=12.0 K 2024-11-24T20:47:53,815 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 1bb3e8397087a4722761784332990ead in 879ms, sequenceid=359, compaction requested=true 2024-11-24T20:47:53,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:53,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:53,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-24T20:47:53,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-24T20:47:53,818 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-24T20:47:53,818 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5670 sec 2024-11-24T20:47:53,819 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.5720 sec 2024-11-24T20:47:53,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 1bb3e8397087a4722761784332990ead 2024-11-24T20:47:53,839 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:47:53,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1bb3e8397087a4722761784332990ead 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-24T20:47:53,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/58e41c204d4445daa821a711dbb1cf57 is 50, key is test_row_0/A:col10/1732481273839/Put/seqid=0 2024-11-24T20:47:53,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:53,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481333845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:53,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:53,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:53,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:53,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481333850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:53,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481333851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:53,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:53,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481333852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:53,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:53,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:53,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:53,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742109_1285 (size=14741) 2024-11-24T20:47:53,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481333956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:53,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:53,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:53,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481333964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:53,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481333965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:53,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481333965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:53,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:54,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481334165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:54,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:54,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:54,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:54,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:54,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481334175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:54,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:54,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481334175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:54,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:54,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:54,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:54,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:54,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481334175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:54,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:54,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:54,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:54,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:54,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:54,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:54,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/58e41c204d4445daa821a711dbb1cf57 2024-11-24T20:47:54,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/2e6e425a08cc40bc84f1dff5f360f2fe is 50, key is test_row_0/B:col10/1732481273839/Put/seqid=0 2024-11-24T20:47:54,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742110_1286 (size=12301) 2024-11-24T20:47:54,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/2e6e425a08cc40bc84f1dff5f360f2fe 2024-11-24T20:47:54,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/d4277cc676a0470d8b7f1993f0b3abb8 is 50, key is test_row_0/C:col10/1732481273839/Put/seqid=0 2024-11-24T20:47:54,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742111_1287 (size=12301) 2024-11-24T20:47:54,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481334475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:54,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:54,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:54,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:54,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481334485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:54,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481334485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:54,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481334485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:54,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:54,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:54,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:54,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/d4277cc676a0470d8b7f1993f0b3abb8 2024-11-24T20:47:54,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/58e41c204d4445daa821a711dbb1cf57 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/58e41c204d4445daa821a711dbb1cf57 2024-11-24T20:47:54,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/58e41c204d4445daa821a711dbb1cf57, entries=200, sequenceid=389, filesize=14.4 K 2024-11-24T20:47:54,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/2e6e425a08cc40bc84f1dff5f360f2fe as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/2e6e425a08cc40bc84f1dff5f360f2fe 2024-11-24T20:47:54,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/2e6e425a08cc40bc84f1dff5f360f2fe, entries=150, sequenceid=389, filesize=12.0 K 2024-11-24T20:47:54,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/d4277cc676a0470d8b7f1993f0b3abb8 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/d4277cc676a0470d8b7f1993f0b3abb8 2024-11-24T20:47:54,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/d4277cc676a0470d8b7f1993f0b3abb8, entries=150, sequenceid=389, filesize=12.0 K 2024-11-24T20:47:54,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=26.84 KB/27480 for 1bb3e8397087a4722761784332990ead in 928ms, sequenceid=389, compaction requested=true 2024-11-24T20:47:54,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:54,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1bb3e8397087a4722761784332990ead:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:47:54,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:54,767 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:47:54,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1bb3e8397087a4722761784332990ead:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:47:54,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:54,767 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:47:54,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1bb3e8397087a4722761784332990ead:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:47:54,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:47:54,770 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:47:54,770 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 1bb3e8397087a4722761784332990ead/B is initiating minor compaction (all files) 2024-11-24T20:47:54,770 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1bb3e8397087a4722761784332990ead/B in TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:54,770 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/bd38d819b1f24afd8c75a7f5ebcb600b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/30a0aef9ae0a4910b678e5860c3d2cba, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/e888621edbd84f0ea0620b7aa4e55a0c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/2e6e425a08cc40bc84f1dff5f360f2fe] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp, totalSize=48.8 K 2024-11-24T20:47:54,770 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57308 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:47:54,770 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 1bb3e8397087a4722761784332990ead/A is initiating minor compaction (all files) 2024-11-24T20:47:54,770 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1bb3e8397087a4722761784332990ead/A in TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:54,771 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/fd3cd083b310435ca6db1e5c1355cd6a, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/60921cf5c8c54d25b5e0c95c6f6325ec, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/f58bcd7cf62d46088856beff0183db4a, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/58e41c204d4445daa821a711dbb1cf57] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp, totalSize=56.0 K 2024-11-24T20:47:54,771 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting bd38d819b1f24afd8c75a7f5ebcb600b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732481271305 2024-11-24T20:47:54,772 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd3cd083b310435ca6db1e5c1355cd6a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732481271305 2024-11-24T20:47:54,772 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 30a0aef9ae0a4910b678e5860c3d2cba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732481271334 2024-11-24T20:47:54,772 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting e888621edbd84f0ea0620b7aa4e55a0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732481271985 2024-11-24T20:47:54,772 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60921cf5c8c54d25b5e0c95c6f6325ec, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732481271334 2024-11-24T20:47:54,773 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e6e425a08cc40bc84f1dff5f360f2fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1732481273838 2024-11-24T20:47:54,775 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting f58bcd7cf62d46088856beff0183db4a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732481271985 2024-11-24T20:47:54,776 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58e41c204d4445daa821a711dbb1cf57, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1732481273202 2024-11-24T20:47:54,786 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1bb3e8397087a4722761784332990ead#B#compaction#248 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:54,787 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/8711997e42c0496aaa6b60c645788479 is 50, key is test_row_0/B:col10/1732481273839/Put/seqid=0 2024-11-24T20:47:54,802 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1bb3e8397087a4722761784332990ead#A#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:54,803 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/c71619fe7855441e869cb5982e804bde is 50, key is test_row_0/A:col10/1732481273839/Put/seqid=0 2024-11-24T20:47:54,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742112_1288 (size=13221) 2024-11-24T20:47:54,842 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/8711997e42c0496aaa6b60c645788479 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/8711997e42c0496aaa6b60c645788479 2024-11-24T20:47:54,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742113_1289 (size=13221) 2024-11-24T20:47:54,855 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/c71619fe7855441e869cb5982e804bde as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/c71619fe7855441e869cb5982e804bde 2024-11-24T20:47:54,856 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1bb3e8397087a4722761784332990ead/B of 1bb3e8397087a4722761784332990ead into 8711997e42c0496aaa6b60c645788479(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:54,856 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:54,856 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., storeName=1bb3e8397087a4722761784332990ead/B, priority=12, startTime=1732481274767; duration=0sec 2024-11-24T20:47:54,856 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:47:54,856 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1bb3e8397087a4722761784332990ead:B 2024-11-24T20:47:54,856 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:47:54,859 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:47:54,859 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 1bb3e8397087a4722761784332990ead/C is initiating minor compaction (all files) 2024-11-24T20:47:54,859 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1bb3e8397087a4722761784332990ead/C in TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:54,859 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/e641cb05d0ed4a2988f78e383abd6796, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/3726977901c948969bcb743d87d4624d, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/9a38fec053bf4271b2f22e3f1ed8f13f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/d4277cc676a0470d8b7f1993f0b3abb8] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp, totalSize=48.8 K 2024-11-24T20:47:54,860 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting e641cb05d0ed4a2988f78e383abd6796, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1732481271305 2024-11-24T20:47:54,860 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 3726977901c948969bcb743d87d4624d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732481271334 2024-11-24T20:47:54,861 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a38fec053bf4271b2f22e3f1ed8f13f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732481271985 2024-11-24T20:47:54,862 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting d4277cc676a0470d8b7f1993f0b3abb8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1732481273838 2024-11-24T20:47:54,864 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1bb3e8397087a4722761784332990ead/A of 1bb3e8397087a4722761784332990ead into c71619fe7855441e869cb5982e804bde(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:54,864 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:54,864 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., storeName=1bb3e8397087a4722761784332990ead/A, priority=12, startTime=1732481274767; duration=0sec 2024-11-24T20:47:54,864 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:54,864 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1bb3e8397087a4722761784332990ead:A 2024-11-24T20:47:54,876 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1bb3e8397087a4722761784332990ead#C#compaction#250 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:54,877 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/54c4144e726b4c4d813f59de8fdf9d6c is 50, key is test_row_0/C:col10/1732481273839/Put/seqid=0 2024-11-24T20:47:54,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742114_1290 (size=13221) 2024-11-24T20:47:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 1bb3e8397087a4722761784332990ead 2024-11-24T20:47:54,992 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:47:54,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1bb3e8397087a4722761784332990ead 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T20:47:54,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/90840ac6ab1f428996d3ed7c648f762b is 50, key is test_row_0/A:col10/1732481274992/Put/seqid=0 2024-11-24T20:47:55,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742115_1291 (size=14741) 2024-11-24T20:47:55,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/90840ac6ab1f428996d3ed7c648f762b 2024-11-24T20:47:55,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/8a6937b5c035463fa4d5ab817a06e0a5 is 50, key is test_row_0/B:col10/1732481274992/Put/seqid=0 2024-11-24T20:47:55,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742116_1292 (size=12301) 2024-11-24T20:47:55,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481335027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:55,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481335028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:55,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:55,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:55,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:55,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481335029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:55,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:55,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:55,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481335030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:55,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:55,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:55,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:55,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:55,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:55,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481335135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:55,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:55,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:55,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:55,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:55,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481335135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:55,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:55,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481335145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:55,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:55,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:55,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:55,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:55,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481335145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:55,299 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/54c4144e726b4c4d813f59de8fdf9d6c as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/54c4144e726b4c4d813f59de8fdf9d6c 2024-11-24T20:47:55,302 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1bb3e8397087a4722761784332990ead/C of 1bb3e8397087a4722761784332990ead into 54c4144e726b4c4d813f59de8fdf9d6c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:55,303 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:55,303 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., storeName=1bb3e8397087a4722761784332990ead/C, priority=12, startTime=1732481274767; duration=0sec 2024-11-24T20:47:55,303 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:55,303 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1bb3e8397087a4722761784332990ead:C 2024-11-24T20:47:55,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:55,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481335345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:55,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:55,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481335345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:55,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:55,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:55,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:55,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:55,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:55,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:55,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:55,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481335355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:55,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:55,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:55,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:55,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481335355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:55,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:55,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:55,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:55,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:55,375 INFO [Thread-996 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-996. 2024-11-24T20:47:55,375 DEBUG [Thread-996 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:47:55,375 INFO [Thread-1002 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1002. 2024-11-24T20:47:55,375 DEBUG [Thread-996 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:55,375 DEBUG [Thread-1002 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:47:55,375 DEBUG [Thread-1002 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:55,375 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:47:55,375 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:47:55,375 DEBUG [Thread-996 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:55,375 DEBUG [Thread-1002 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:55,376 INFO [Thread-1000 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1000. 2024-11-24T20:47:55,376 DEBUG [Thread-1000 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:47:55,376 DEBUG [Thread-1000 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:55,376 DEBUG [Thread-1000 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:55,376 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:47:55,378 INFO [Thread-1004 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1004. 2024-11-24T20:47:55,378 DEBUG [Thread-1004 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:47:55,378 DEBUG [Thread-1004 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:55,378 DEBUG [Thread-1004 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:55,378 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:47:55,378 INFO [Thread-998 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-998. 2024-11-24T20:47:55,378 DEBUG [Thread-998 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:47:55,378 DEBUG [Thread-998 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:55,378 DEBUG [Thread-998 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:55,378 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:47:55,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T20:47:55,416 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:47:55,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/8a6937b5c035463fa4d5ab817a06e0a5 2024-11-24T20:47:55,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/6e38b2efd10640ee8d5113cc914c7ff1 is 50, key is test_row_0/C:col10/1732481274992/Put/seqid=0 2024-11-24T20:47:55,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742117_1293 (size=12301) 2024-11-24T20:47:55,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:55,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42302 deadline: 1732481335655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42250 deadline: 1732481335655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:55,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:55,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:55,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:55,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:55,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:55,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:55,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:55,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:47:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42264 deadline: 1732481335665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42288 deadline: 1732481335665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:47:55,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:55,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:55,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:55,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:55,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:47:55,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:47:55,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/6e38b2efd10640ee8d5113cc914c7ff1 2024-11-24T20:47:55,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/90840ac6ab1f428996d3ed7c648f762b as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/90840ac6ab1f428996d3ed7c648f762b 2024-11-24T20:47:55,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/90840ac6ab1f428996d3ed7c648f762b, entries=200, sequenceid=402, filesize=14.4 K 2024-11-24T20:47:55,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/8a6937b5c035463fa4d5ab817a06e0a5 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/8a6937b5c035463fa4d5ab817a06e0a5 2024-11-24T20:47:55,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/8a6937b5c035463fa4d5ab817a06e0a5, entries=150, sequenceid=402, filesize=12.0 K 2024-11-24T20:47:55,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/6e38b2efd10640ee8d5113cc914c7ff1 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/6e38b2efd10640ee8d5113cc914c7ff1 2024-11-24T20:47:55,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/6e38b2efd10640ee8d5113cc914c7ff1, entries=150, sequenceid=402, filesize=12.0 K 2024-11-24T20:47:55,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1bb3e8397087a4722761784332990ead in 878ms, sequenceid=402, compaction requested=false 2024-11-24T20:47:55,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:56,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 1bb3e8397087a4722761784332990ead 2024-11-24T20:47:56,167 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:47:56,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1bb3e8397087a4722761784332990ead 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-24T20:47:56,167 INFO [Thread-985 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-985. 2024-11-24T20:47:56,167 INFO [Thread-989 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-989. 2024-11-24T20:47:56,167 DEBUG [Thread-985 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:47:56,167 DEBUG [Thread-989 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:47:56,167 DEBUG [Thread-985 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:56,167 DEBUG [Thread-989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:56,167 DEBUG [Thread-985 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:56,167 DEBUG [Thread-989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:56,167 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:47:56,168 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:47:56,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/078a81b8c06c4954adf1d228bbdead13 is 50, key is test_row_0/A:col10/1732481276165/Put/seqid=0 2024-11-24T20:47:56,177 INFO [Thread-993 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-993. 2024-11-24T20:47:56,177 INFO [Thread-987 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-987. 2024-11-24T20:47:56,177 DEBUG [Thread-993 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:47:56,177 DEBUG [Thread-993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:56,177 DEBUG [Thread-987 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:47:56,177 DEBUG [Thread-987 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:56,177 DEBUG [Thread-993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:56,177 DEBUG [Thread-987 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:47:56,177 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:47:56,178 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:47:56,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742118_1294 (size=12301) 2024-11-24T20:47:56,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/078a81b8c06c4954adf1d228bbdead13 2024-11-24T20:47:56,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/86ad180410504f759302d663f2283740 is 50, key is test_row_0/B:col10/1732481276165/Put/seqid=0 2024-11-24T20:47:56,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742119_1295 (size=12301) 2024-11-24T20:47:56,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/86ad180410504f759302d663f2283740 2024-11-24T20:47:56,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/82bdeb8b7ecf47738fb49aee2c215ce4 is 50, key is test_row_0/C:col10/1732481276165/Put/seqid=0 2024-11-24T20:47:56,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742120_1296 (size=12301) 2024-11-24T20:47:56,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/82bdeb8b7ecf47738fb49aee2c215ce4 2024-11-24T20:47:56,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/078a81b8c06c4954adf1d228bbdead13 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/078a81b8c06c4954adf1d228bbdead13 2024-11-24T20:47:56,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/078a81b8c06c4954adf1d228bbdead13, entries=150, sequenceid=430, filesize=12.0 K 2024-11-24T20:47:56,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/86ad180410504f759302d663f2283740 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/86ad180410504f759302d663f2283740 2024-11-24T20:47:56,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/86ad180410504f759302d663f2283740, entries=150, sequenceid=430, filesize=12.0 K 2024-11-24T20:47:56,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/82bdeb8b7ecf47738fb49aee2c215ce4 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/82bdeb8b7ecf47738fb49aee2c215ce4 2024-11-24T20:47:56,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/82bdeb8b7ecf47738fb49aee2c215ce4, entries=150, sequenceid=430, filesize=12.0 K 2024-11-24T20:47:56,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=13.42 KB/13740 for 1bb3e8397087a4722761784332990ead in 480ms, sequenceid=430, compaction requested=true 2024-11-24T20:47:56,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:56,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1bb3e8397087a4722761784332990ead:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:47:56,647 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:47:56,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:56,647 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:47:56,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1bb3e8397087a4722761784332990ead:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:47:56,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:56,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1bb3e8397087a4722761784332990ead:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:47:56,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:47:56,648 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:47:56,648 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:47:56,648 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 1bb3e8397087a4722761784332990ead/A is initiating minor compaction (all files) 2024-11-24T20:47:56,648 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 1bb3e8397087a4722761784332990ead/B is initiating minor compaction (all files) 2024-11-24T20:47:56,648 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1bb3e8397087a4722761784332990ead/B in TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:56,648 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/8711997e42c0496aaa6b60c645788479, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/8a6937b5c035463fa4d5ab817a06e0a5, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/86ad180410504f759302d663f2283740] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp, totalSize=36.9 K 2024-11-24T20:47:56,648 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1bb3e8397087a4722761784332990ead/A in TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:56,648 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/c71619fe7855441e869cb5982e804bde, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/90840ac6ab1f428996d3ed7c648f762b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/078a81b8c06c4954adf1d228bbdead13] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp, totalSize=39.3 K 2024-11-24T20:47:56,649 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 8711997e42c0496aaa6b60c645788479, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1732481273838 2024-11-24T20:47:56,649 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting c71619fe7855441e869cb5982e804bde, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1732481273838 2024-11-24T20:47:56,649 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a6937b5c035463fa4d5ab817a06e0a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732481273849 2024-11-24T20:47:56,649 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90840ac6ab1f428996d3ed7c648f762b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732481273849 2024-11-24T20:47:56,649 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 86ad180410504f759302d663f2283740, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732481275027 2024-11-24T20:47:56,649 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 078a81b8c06c4954adf1d228bbdead13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732481275027 2024-11-24T20:47:56,658 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1bb3e8397087a4722761784332990ead#B#compaction#257 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:56,658 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/16901d78774c4b28a23e7507d314a2c2 is 50, key is test_row_0/B:col10/1732481276165/Put/seqid=0 2024-11-24T20:47:56,666 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1bb3e8397087a4722761784332990ead#A#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:56,667 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/dc847360ae7f4ac4bf405f4b84277164 is 50, key is test_row_0/A:col10/1732481276165/Put/seqid=0 2024-11-24T20:47:56,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742121_1297 (size=13323) 2024-11-24T20:47:56,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742122_1298 (size=13323) 2024-11-24T20:47:56,677 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/16901d78774c4b28a23e7507d314a2c2 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/16901d78774c4b28a23e7507d314a2c2 2024-11-24T20:47:56,681 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1bb3e8397087a4722761784332990ead/B of 1bb3e8397087a4722761784332990ead into 16901d78774c4b28a23e7507d314a2c2(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:56,681 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:56,681 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., storeName=1bb3e8397087a4722761784332990ead/B, priority=13, startTime=1732481276647; duration=0sec 2024-11-24T20:47:56,681 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:47:56,681 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1bb3e8397087a4722761784332990ead:B 2024-11-24T20:47:56,681 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:47:56,682 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:47:56,682 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 1bb3e8397087a4722761784332990ead/C is initiating minor compaction (all files) 2024-11-24T20:47:56,682 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1bb3e8397087a4722761784332990ead/C in TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:47:56,682 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/54c4144e726b4c4d813f59de8fdf9d6c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/6e38b2efd10640ee8d5113cc914c7ff1, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/82bdeb8b7ecf47738fb49aee2c215ce4] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp, totalSize=36.9 K 2024-11-24T20:47:56,682 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 54c4144e726b4c4d813f59de8fdf9d6c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1732481273838 2024-11-24T20:47:56,683 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e38b2efd10640ee8d5113cc914c7ff1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732481273849 2024-11-24T20:47:56,683 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 82bdeb8b7ecf47738fb49aee2c215ce4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732481275027 2024-11-24T20:47:56,690 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1bb3e8397087a4722761784332990ead#C#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:47:56,691 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/c6a2364a8de440e49010a81578fca086 is 50, key is test_row_0/C:col10/1732481276165/Put/seqid=0 2024-11-24T20:47:56,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742123_1299 (size=13323) 2024-11-24T20:47:57,089 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/dc847360ae7f4ac4bf405f4b84277164 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/dc847360ae7f4ac4bf405f4b84277164 2024-11-24T20:47:57,095 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1bb3e8397087a4722761784332990ead/A of 1bb3e8397087a4722761784332990ead into dc847360ae7f4ac4bf405f4b84277164(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:57,095 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:57,095 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., storeName=1bb3e8397087a4722761784332990ead/A, priority=13, startTime=1732481276647; duration=0sec 2024-11-24T20:47:57,095 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:57,095 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1bb3e8397087a4722761784332990ead:A 2024-11-24T20:47:57,101 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/c6a2364a8de440e49010a81578fca086 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/c6a2364a8de440e49010a81578fca086 2024-11-24T20:47:57,106 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1bb3e8397087a4722761784332990ead/C of 1bb3e8397087a4722761784332990ead into c6a2364a8de440e49010a81578fca086(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:47:57,106 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1bb3e8397087a4722761784332990ead: 2024-11-24T20:47:57,106 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., storeName=1bb3e8397087a4722761784332990ead/C, priority=13, startTime=1732481276647; duration=0sec 2024-11-24T20:47:57,106 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:47:57,106 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1bb3e8397087a4722761784332990ead:C 2024-11-24T20:48:03,770 INFO [Thread-991 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-991. 2024-11-24T20:48:03,770 DEBUG [Thread-991 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:48:03,770 DEBUG [Thread-991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:03,770 DEBUG [Thread-991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:03,770 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:48:03,771 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T20:48:03,771 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-24T20:48:03,771 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-24T20:48:03,771 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-24T20:48:03,771 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-11-24T20:48:03,771 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 114 2024-11-24T20:48:03,771 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T20:48:03,771 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4232 2024-11-24T20:48:03,771 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4353 2024-11-24T20:48:03,771 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4136 2024-11-24T20:48:03,771 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4244 2024-11-24T20:48:03,771 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4359 2024-11-24T20:48:03,771 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T20:48:03,772 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T20:48:03,772 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T20:48:03,772 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:03,772 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:03,772 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:48:03,773 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T20:48:03,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T20:48:03,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-24T20:48:03,778 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732481283778"}]},"ts":"1732481283778"} 2024-11-24T20:48:03,780 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T20:48:03,780 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T20:48:03,781 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T20:48:03,783 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1bb3e8397087a4722761784332990ead, UNASSIGN}] 2024-11-24T20:48:03,784 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1bb3e8397087a4722761784332990ead, UNASSIGN 2024-11-24T20:48:03,785 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=1bb3e8397087a4722761784332990ead, regionState=CLOSING, regionLocation=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:03,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1bb3e8397087a4722761784332990ead, UNASSIGN because future has completed 2024-11-24T20:48:03,787 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T20:48:03,787 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291}] 2024-11-24T20:48:03,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-24T20:48:03,940 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close 1bb3e8397087a4722761784332990ead 2024-11-24T20:48:03,940 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T20:48:03,940 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing 1bb3e8397087a4722761784332990ead, disabling compactions & flushes 2024-11-24T20:48:03,940 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:48:03,940 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:48:03,940 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. after waiting 0 ms 2024-11-24T20:48:03,940 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:48:03,941 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(2902): Flushing 1bb3e8397087a4722761784332990ead 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-24T20:48:03,945 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/e0712451d98a46b7a1d008e1da6a4c4b is 50, key is test_row_1/A:col10/1732481283766/Put/seqid=0 2024-11-24T20:48:03,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742124_1300 (size=9857) 2024-11-24T20:48:04,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-24T20:48:04,351 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/e0712451d98a46b7a1d008e1da6a4c4b 2024-11-24T20:48:04,358 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/2dbeda9b9b3044449aad7c7511c9faa9 is 50, key is test_row_1/B:col10/1732481283766/Put/seqid=0 2024-11-24T20:48:04,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742125_1301 (size=9857) 2024-11-24T20:48:04,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-24T20:48:04,764 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/2dbeda9b9b3044449aad7c7511c9faa9 2024-11-24T20:48:04,777 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/4a542f09fd9d4126a121feeefcdea5a0 is 50, key is test_row_1/C:col10/1732481283766/Put/seqid=0 2024-11-24T20:48:04,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742126_1302 (size=9857) 2024-11-24T20:48:04,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-24T20:48:05,183 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/4a542f09fd9d4126a121feeefcdea5a0 2024-11-24T20:48:05,187 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/A/e0712451d98a46b7a1d008e1da6a4c4b as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/e0712451d98a46b7a1d008e1da6a4c4b 2024-11-24T20:48:05,190 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/e0712451d98a46b7a1d008e1da6a4c4b, entries=100, sequenceid=439, filesize=9.6 K 2024-11-24T20:48:05,191 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/B/2dbeda9b9b3044449aad7c7511c9faa9 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/2dbeda9b9b3044449aad7c7511c9faa9 2024-11-24T20:48:05,194 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/2dbeda9b9b3044449aad7c7511c9faa9, entries=100, sequenceid=439, filesize=9.6 K 2024-11-24T20:48:05,195 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/.tmp/C/4a542f09fd9d4126a121feeefcdea5a0 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/4a542f09fd9d4126a121feeefcdea5a0 2024-11-24T20:48:05,199 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/4a542f09fd9d4126a121feeefcdea5a0, entries=100, sequenceid=439, filesize=9.6 K 2024-11-24T20:48:05,199 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 1bb3e8397087a4722761784332990ead in 1259ms, sequenceid=439, compaction requested=false 2024-11-24T20:48:05,200 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/d3658e55f4784d74b6219e6b4368b58f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/fbec8a44dfb34d9294a29fa9950a789b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/da0e223ebc9a4c54b5e46104dcbda621, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/a30faf1ff6be4abf93a6695892c0d807, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/18172e16ebd145c9842b515020eca86f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/9692348eae134b208e34555a3a609bf7, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/2c3f228644ff498babadbd0491d67800, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/72bd01b05e4347f8866dfaa28dd3d3c7, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/0b4d31beebcb4e23aa2ecee9582bed3d, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/9668898378cc491f8644a20a9d4cee7b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/97989a481d084ed2bc0c72d6cd381182, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/42f94f7f00da45bba089bcdcabffd03c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/e9f9b4208122436caa689ec15af57f89, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/60fc230f575c4b619dac0a3c90530768, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/d5eb5a6130b446199761faa52a3af359, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/b538838a0c8a4ebc94d0c6a53edeb501, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/97bec558ad1f460f8d7f924d54ecc275, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/261813ba9dba45ef8a0f0517afe4f1f3, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/f60445ee0241477ab00d0c9821c19a60, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/a656707f14d84e39ac5db00b5c79a469, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/e2b43012d9b941e6a3a43bd6737dbe0b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/411143cd754742b5971f0baba1a5facd, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/fd3cd083b310435ca6db1e5c1355cd6a, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/9be698c9bd02436b8a7ad209256aada6, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/60921cf5c8c54d25b5e0c95c6f6325ec, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/f58bcd7cf62d46088856beff0183db4a, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/58e41c204d4445daa821a711dbb1cf57, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/c71619fe7855441e869cb5982e804bde, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/90840ac6ab1f428996d3ed7c648f762b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/078a81b8c06c4954adf1d228bbdead13] to archive 2024-11-24T20:48:05,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T20:48:05,202 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/d3658e55f4784d74b6219e6b4368b58f to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/d3658e55f4784d74b6219e6b4368b58f 2024-11-24T20:48:05,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/fbec8a44dfb34d9294a29fa9950a789b to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/fbec8a44dfb34d9294a29fa9950a789b 2024-11-24T20:48:05,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/da0e223ebc9a4c54b5e46104dcbda621 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/da0e223ebc9a4c54b5e46104dcbda621 2024-11-24T20:48:05,205 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/a30faf1ff6be4abf93a6695892c0d807 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/a30faf1ff6be4abf93a6695892c0d807 2024-11-24T20:48:05,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/18172e16ebd145c9842b515020eca86f to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/18172e16ebd145c9842b515020eca86f 2024-11-24T20:48:05,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/9692348eae134b208e34555a3a609bf7 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/9692348eae134b208e34555a3a609bf7 2024-11-24T20:48:05,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/2c3f228644ff498babadbd0491d67800 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/2c3f228644ff498babadbd0491d67800 2024-11-24T20:48:05,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/72bd01b05e4347f8866dfaa28dd3d3c7 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/72bd01b05e4347f8866dfaa28dd3d3c7 2024-11-24T20:48:05,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/0b4d31beebcb4e23aa2ecee9582bed3d to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/0b4d31beebcb4e23aa2ecee9582bed3d 2024-11-24T20:48:05,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/9668898378cc491f8644a20a9d4cee7b to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/9668898378cc491f8644a20a9d4cee7b 2024-11-24T20:48:05,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/97989a481d084ed2bc0c72d6cd381182 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/97989a481d084ed2bc0c72d6cd381182 2024-11-24T20:48:05,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/42f94f7f00da45bba089bcdcabffd03c to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/42f94f7f00da45bba089bcdcabffd03c 2024-11-24T20:48:05,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/e9f9b4208122436caa689ec15af57f89 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/e9f9b4208122436caa689ec15af57f89 2024-11-24T20:48:05,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/60fc230f575c4b619dac0a3c90530768 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/60fc230f575c4b619dac0a3c90530768 2024-11-24T20:48:05,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/d5eb5a6130b446199761faa52a3af359 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/d5eb5a6130b446199761faa52a3af359 2024-11-24T20:48:05,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/b538838a0c8a4ebc94d0c6a53edeb501 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/b538838a0c8a4ebc94d0c6a53edeb501 2024-11-24T20:48:05,217 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/97bec558ad1f460f8d7f924d54ecc275 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/97bec558ad1f460f8d7f924d54ecc275 2024-11-24T20:48:05,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/261813ba9dba45ef8a0f0517afe4f1f3 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/261813ba9dba45ef8a0f0517afe4f1f3 2024-11-24T20:48:05,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/f60445ee0241477ab00d0c9821c19a60 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/f60445ee0241477ab00d0c9821c19a60 2024-11-24T20:48:05,219 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/a656707f14d84e39ac5db00b5c79a469 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/a656707f14d84e39ac5db00b5c79a469 2024-11-24T20:48:05,220 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/e2b43012d9b941e6a3a43bd6737dbe0b to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/e2b43012d9b941e6a3a43bd6737dbe0b 2024-11-24T20:48:05,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/411143cd754742b5971f0baba1a5facd to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/411143cd754742b5971f0baba1a5facd 2024-11-24T20:48:05,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/fd3cd083b310435ca6db1e5c1355cd6a to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/fd3cd083b310435ca6db1e5c1355cd6a 2024-11-24T20:48:05,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/9be698c9bd02436b8a7ad209256aada6 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/9be698c9bd02436b8a7ad209256aada6 2024-11-24T20:48:05,223 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/60921cf5c8c54d25b5e0c95c6f6325ec to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/60921cf5c8c54d25b5e0c95c6f6325ec 2024-11-24T20:48:05,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/f58bcd7cf62d46088856beff0183db4a to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/f58bcd7cf62d46088856beff0183db4a 2024-11-24T20:48:05,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/58e41c204d4445daa821a711dbb1cf57 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/58e41c204d4445daa821a711dbb1cf57 2024-11-24T20:48:05,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/c71619fe7855441e869cb5982e804bde to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/c71619fe7855441e869cb5982e804bde 2024-11-24T20:48:05,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/90840ac6ab1f428996d3ed7c648f762b to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/90840ac6ab1f428996d3ed7c648f762b 2024-11-24T20:48:05,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/078a81b8c06c4954adf1d228bbdead13 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/078a81b8c06c4954adf1d228bbdead13 2024-11-24T20:48:05,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/e2947a2a81e144b8b15133e80dc51f76, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/e3a24f74823b49489c9ea82ac8f6c02b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/cb910dba1e54481bbf10f75a2bcafcd4, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/feec15f5a8c54cf4a72910f558a5fe5b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/659cc4d46a12496fad48dfeecd580eb0, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/c9c74ce5ad194289bf4775f1d708e99a, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/8fde4f32c3c44b7ab4c7df9124ff612d, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/03d8240044ab46e5be9ac7af8a985692, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/c061b759469e491baf8ae5c73965265f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/8e38a712026d452195a06ffe85872191, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/1537e62c871a4bb09e985e7f9709d373, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/d50706098f0f4528a220e25a0bc32463, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/731479cf9a7946aba59bf766f3d2e2a2, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/517492a1a91d4576add1ac362acf5327, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/62fcedcc075e4b228033d7f648918e2c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/94aa7b6338e44f989603341dc43bc466, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/b5451853645e4de58303d7f49a0ab2f4, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/b8fb0cb364bc4dfebcf0ef07b627912f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/f1f67e366c774e97aee2d168900621d3, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/81fb258006464f2eb9b4a80ccb07b1d5, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/2d3adc2a958a46e39fc8ba65ea8da9f8, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/3eba0f9f3d3d4fd2ae5852874878c03d, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/bd38d819b1f24afd8c75a7f5ebcb600b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/07e7cefd293a45069fce2d0a6778cb92, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/30a0aef9ae0a4910b678e5860c3d2cba, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/e888621edbd84f0ea0620b7aa4e55a0c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/8711997e42c0496aaa6b60c645788479, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/2e6e425a08cc40bc84f1dff5f360f2fe, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/8a6937b5c035463fa4d5ab817a06e0a5, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/86ad180410504f759302d663f2283740] to archive 2024-11-24T20:48:05,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T20:48:05,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/e2947a2a81e144b8b15133e80dc51f76 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/e2947a2a81e144b8b15133e80dc51f76 2024-11-24T20:48:05,231 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/e3a24f74823b49489c9ea82ac8f6c02b to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/e3a24f74823b49489c9ea82ac8f6c02b 2024-11-24T20:48:05,232 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/cb910dba1e54481bbf10f75a2bcafcd4 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/cb910dba1e54481bbf10f75a2bcafcd4 2024-11-24T20:48:05,232 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/feec15f5a8c54cf4a72910f558a5fe5b to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/feec15f5a8c54cf4a72910f558a5fe5b 2024-11-24T20:48:05,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/659cc4d46a12496fad48dfeecd580eb0 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/659cc4d46a12496fad48dfeecd580eb0 2024-11-24T20:48:05,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/c9c74ce5ad194289bf4775f1d708e99a to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/c9c74ce5ad194289bf4775f1d708e99a 2024-11-24T20:48:05,235 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/8fde4f32c3c44b7ab4c7df9124ff612d to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/8fde4f32c3c44b7ab4c7df9124ff612d 2024-11-24T20:48:05,236 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/03d8240044ab46e5be9ac7af8a985692 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/03d8240044ab46e5be9ac7af8a985692 2024-11-24T20:48:05,236 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/c061b759469e491baf8ae5c73965265f to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/c061b759469e491baf8ae5c73965265f 2024-11-24T20:48:05,237 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/8e38a712026d452195a06ffe85872191 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/8e38a712026d452195a06ffe85872191 2024-11-24T20:48:05,238 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/1537e62c871a4bb09e985e7f9709d373 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/1537e62c871a4bb09e985e7f9709d373 2024-11-24T20:48:05,239 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/d50706098f0f4528a220e25a0bc32463 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/d50706098f0f4528a220e25a0bc32463 2024-11-24T20:48:05,240 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/731479cf9a7946aba59bf766f3d2e2a2 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/731479cf9a7946aba59bf766f3d2e2a2 2024-11-24T20:48:05,241 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/517492a1a91d4576add1ac362acf5327 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/517492a1a91d4576add1ac362acf5327 2024-11-24T20:48:05,242 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/62fcedcc075e4b228033d7f648918e2c to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/62fcedcc075e4b228033d7f648918e2c 2024-11-24T20:48:05,242 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/94aa7b6338e44f989603341dc43bc466 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/94aa7b6338e44f989603341dc43bc466 2024-11-24T20:48:05,243 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/b5451853645e4de58303d7f49a0ab2f4 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/b5451853645e4de58303d7f49a0ab2f4 2024-11-24T20:48:05,244 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/b8fb0cb364bc4dfebcf0ef07b627912f to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/b8fb0cb364bc4dfebcf0ef07b627912f 2024-11-24T20:48:05,245 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/f1f67e366c774e97aee2d168900621d3 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/f1f67e366c774e97aee2d168900621d3 2024-11-24T20:48:05,247 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/81fb258006464f2eb9b4a80ccb07b1d5 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/81fb258006464f2eb9b4a80ccb07b1d5 2024-11-24T20:48:05,248 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/2d3adc2a958a46e39fc8ba65ea8da9f8 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/2d3adc2a958a46e39fc8ba65ea8da9f8 2024-11-24T20:48:05,249 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/3eba0f9f3d3d4fd2ae5852874878c03d to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/3eba0f9f3d3d4fd2ae5852874878c03d 2024-11-24T20:48:05,249 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/bd38d819b1f24afd8c75a7f5ebcb600b to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/bd38d819b1f24afd8c75a7f5ebcb600b 2024-11-24T20:48:05,250 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/07e7cefd293a45069fce2d0a6778cb92 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/07e7cefd293a45069fce2d0a6778cb92 2024-11-24T20:48:05,251 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/30a0aef9ae0a4910b678e5860c3d2cba to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/30a0aef9ae0a4910b678e5860c3d2cba 2024-11-24T20:48:05,252 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/e888621edbd84f0ea0620b7aa4e55a0c to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/e888621edbd84f0ea0620b7aa4e55a0c 2024-11-24T20:48:05,252 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/8711997e42c0496aaa6b60c645788479 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/8711997e42c0496aaa6b60c645788479 2024-11-24T20:48:05,253 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/2e6e425a08cc40bc84f1dff5f360f2fe to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/2e6e425a08cc40bc84f1dff5f360f2fe 2024-11-24T20:48:05,254 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/8a6937b5c035463fa4d5ab817a06e0a5 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/8a6937b5c035463fa4d5ab817a06e0a5 2024-11-24T20:48:05,254 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/86ad180410504f759302d663f2283740 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/86ad180410504f759302d663f2283740 2024-11-24T20:48:05,256 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/084cf5f2ee664988b479c3b2eecb9339, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/119a666dc3994198b7cfdab4cd457350, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/aed2954fa4184f828b9372f1198ce7cd, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/883afc54381c4b7893ff01464d81eec2, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/5d532df640384b549c7c193acbc8a129, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/bc5433afd90c49d6b8219c9fa159d0e6, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/3ed03bd1a58848bb8b499c1ada852220, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/4d70fd7745614ae485465c8f672616b5, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/7d0aec188b9842119d50de7d11bcc755, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/66fe8e73b5ba47728c819d0d728bd2bb, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/eda703877aa44f3f843f16593d2d20c1, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/7b258327786a4077a5c7d411e2266a99, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/4b76c9f7955a4e7382b3d5a866ce7e62, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/727573c4159547f0aaabaeae9b0e4db0, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/67d9e3c4738e4cd099638408deaf6008, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/b0f95e10e8834d3a8d98cce743e6866f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/026fa4275ec44e549501eb5f77fda006, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/f2c8c55cd9d347db9ec63c35d73a1e9b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/c4f5826051d1471483af4f414bd9d6f1, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/03bb999b8cf442fda5dce3d3a614f1ce, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/5cbc721bdccf4ee59d8827a7500cb75e, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/ded90923c5694efebee1fa156b6ed932, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/e641cb05d0ed4a2988f78e383abd6796, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/ec7cdc394592499ba29170285a3b393c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/3726977901c948969bcb743d87d4624d, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/9a38fec053bf4271b2f22e3f1ed8f13f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/54c4144e726b4c4d813f59de8fdf9d6c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/d4277cc676a0470d8b7f1993f0b3abb8, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/6e38b2efd10640ee8d5113cc914c7ff1, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/82bdeb8b7ecf47738fb49aee2c215ce4] to archive 2024-11-24T20:48:05,257 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T20:48:05,258 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/084cf5f2ee664988b479c3b2eecb9339 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/084cf5f2ee664988b479c3b2eecb9339 2024-11-24T20:48:05,259 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/119a666dc3994198b7cfdab4cd457350 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/119a666dc3994198b7cfdab4cd457350 2024-11-24T20:48:05,260 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/aed2954fa4184f828b9372f1198ce7cd to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/aed2954fa4184f828b9372f1198ce7cd 2024-11-24T20:48:05,262 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/883afc54381c4b7893ff01464d81eec2 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/883afc54381c4b7893ff01464d81eec2 2024-11-24T20:48:05,263 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/5d532df640384b549c7c193acbc8a129 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/5d532df640384b549c7c193acbc8a129 2024-11-24T20:48:05,264 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/bc5433afd90c49d6b8219c9fa159d0e6 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/bc5433afd90c49d6b8219c9fa159d0e6 2024-11-24T20:48:05,265 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/3ed03bd1a58848bb8b499c1ada852220 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/3ed03bd1a58848bb8b499c1ada852220 2024-11-24T20:48:05,267 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/4d70fd7745614ae485465c8f672616b5 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/4d70fd7745614ae485465c8f672616b5 2024-11-24T20:48:05,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/7d0aec188b9842119d50de7d11bcc755 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/7d0aec188b9842119d50de7d11bcc755 2024-11-24T20:48:05,269 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/66fe8e73b5ba47728c819d0d728bd2bb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/66fe8e73b5ba47728c819d0d728bd2bb 2024-11-24T20:48:05,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/eda703877aa44f3f843f16593d2d20c1 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/eda703877aa44f3f843f16593d2d20c1 2024-11-24T20:48:05,272 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/7b258327786a4077a5c7d411e2266a99 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/7b258327786a4077a5c7d411e2266a99 2024-11-24T20:48:05,273 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/4b76c9f7955a4e7382b3d5a866ce7e62 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/4b76c9f7955a4e7382b3d5a866ce7e62 2024-11-24T20:48:05,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/727573c4159547f0aaabaeae9b0e4db0 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/727573c4159547f0aaabaeae9b0e4db0 2024-11-24T20:48:05,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/67d9e3c4738e4cd099638408deaf6008 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/67d9e3c4738e4cd099638408deaf6008 2024-11-24T20:48:05,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/b0f95e10e8834d3a8d98cce743e6866f to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/b0f95e10e8834d3a8d98cce743e6866f 2024-11-24T20:48:05,277 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/026fa4275ec44e549501eb5f77fda006 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/026fa4275ec44e549501eb5f77fda006 2024-11-24T20:48:05,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/f2c8c55cd9d347db9ec63c35d73a1e9b to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/f2c8c55cd9d347db9ec63c35d73a1e9b 2024-11-24T20:48:05,280 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/c4f5826051d1471483af4f414bd9d6f1 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/c4f5826051d1471483af4f414bd9d6f1 2024-11-24T20:48:05,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/03bb999b8cf442fda5dce3d3a614f1ce to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/03bb999b8cf442fda5dce3d3a614f1ce 2024-11-24T20:48:05,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/5cbc721bdccf4ee59d8827a7500cb75e to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/5cbc721bdccf4ee59d8827a7500cb75e 2024-11-24T20:48:05,283 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/ded90923c5694efebee1fa156b6ed932 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/ded90923c5694efebee1fa156b6ed932 2024-11-24T20:48:05,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/e641cb05d0ed4a2988f78e383abd6796 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/e641cb05d0ed4a2988f78e383abd6796 2024-11-24T20:48:05,285 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/ec7cdc394592499ba29170285a3b393c to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/ec7cdc394592499ba29170285a3b393c 2024-11-24T20:48:05,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/3726977901c948969bcb743d87d4624d to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/3726977901c948969bcb743d87d4624d 2024-11-24T20:48:05,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/9a38fec053bf4271b2f22e3f1ed8f13f to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/9a38fec053bf4271b2f22e3f1ed8f13f 2024-11-24T20:48:05,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/54c4144e726b4c4d813f59de8fdf9d6c to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/54c4144e726b4c4d813f59de8fdf9d6c 2024-11-24T20:48:05,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/d4277cc676a0470d8b7f1993f0b3abb8 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/d4277cc676a0470d8b7f1993f0b3abb8 2024-11-24T20:48:05,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/6e38b2efd10640ee8d5113cc914c7ff1 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/6e38b2efd10640ee8d5113cc914c7ff1 2024-11-24T20:48:05,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/82bdeb8b7ecf47738fb49aee2c215ce4 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/82bdeb8b7ecf47738fb49aee2c215ce4 2024-11-24T20:48:05,296 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/recovered.edits/442.seqid, newMaxSeqId=442, maxSeqId=1 2024-11-24T20:48:05,296 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. 2024-11-24T20:48:05,297 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for 1bb3e8397087a4722761784332990ead: Waiting for close lock at 1732481283940Running coprocessor pre-close hooks at 1732481283940Disabling compacts and flushes for region at 1732481283940Disabling writes for close at 1732481283940Obtaining lock to block concurrent updates at 1732481283941 (+1 ms)Preparing flush snapshotting stores in 1bb3e8397087a4722761784332990ead at 1732481283941Finished memstore snapshotting TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1732481283941Flushing stores of TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead. at 1732481283941Flushing 1bb3e8397087a4722761784332990ead/A: creating writer at 1732481283941Flushing 1bb3e8397087a4722761784332990ead/A: appending metadata at 1732481283944 (+3 ms)Flushing 1bb3e8397087a4722761784332990ead/A: closing flushed file at 1732481283944Flushing 1bb3e8397087a4722761784332990ead/B: creating writer at 1732481284355 (+411 ms)Flushing 1bb3e8397087a4722761784332990ead/B: appending metadata at 1732481284358 (+3 ms)Flushing 1bb3e8397087a4722761784332990ead/B: closing flushed file at 1732481284358Flushing 1bb3e8397087a4722761784332990ead/C: creating writer at 1732481284772 (+414 ms)Flushing 1bb3e8397087a4722761784332990ead/C: appending metadata at 1732481284776 (+4 ms)Flushing 1bb3e8397087a4722761784332990ead/C: closing flushed file at 1732481284776Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69e46293: reopening flushed file at 1732481285186 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30679daf: reopening flushed file at 1732481285190 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25b5246b: reopening flushed file at 1732481285194 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 1bb3e8397087a4722761784332990ead in 1259ms, sequenceid=439, compaction requested=false at 1732481285199 (+5 ms)Writing region close event to WAL at 1732481285293 (+94 ms)Running coprocessor post-close hooks at 1732481285296 (+3 ms)Closed at 1732481285296 2024-11-24T20:48:05,298 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed 1bb3e8397087a4722761784332990ead 2024-11-24T20:48:05,299 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=1bb3e8397087a4722761784332990ead, regionState=CLOSED 2024-11-24T20:48:05,300 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 because future has completed 2024-11-24T20:48:05,302 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-24T20:48:05,302 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure 1bb3e8397087a4722761784332990ead, server=20b981ef5c27,34625,1732481182291 in 1.5140 sec 2024-11-24T20:48:05,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-24T20:48:05,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1bb3e8397087a4722761784332990ead, UNASSIGN in 1.5190 sec 2024-11-24T20:48:05,305 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-24T20:48:05,305 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5220 sec 2024-11-24T20:48:05,306 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732481285306"}]},"ts":"1732481285306"} 2024-11-24T20:48:05,307 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T20:48:05,307 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T20:48:05,308 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5330 sec 2024-11-24T20:48:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-24T20:48:05,925 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:48:05,926 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T20:48:05,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:48:05,927 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:48:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-24T20:48:05,928 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:48:05,930 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead 2024-11-24T20:48:05,933 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A, FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B, FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C, FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/recovered.edits] 2024-11-24T20:48:05,935 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/dc847360ae7f4ac4bf405f4b84277164 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/dc847360ae7f4ac4bf405f4b84277164 2024-11-24T20:48:05,936 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/e0712451d98a46b7a1d008e1da6a4c4b to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/A/e0712451d98a46b7a1d008e1da6a4c4b 2024-11-24T20:48:05,938 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/16901d78774c4b28a23e7507d314a2c2 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/16901d78774c4b28a23e7507d314a2c2 2024-11-24T20:48:05,938 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/2dbeda9b9b3044449aad7c7511c9faa9 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/B/2dbeda9b9b3044449aad7c7511c9faa9 2024-11-24T20:48:05,940 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/4a542f09fd9d4126a121feeefcdea5a0 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/4a542f09fd9d4126a121feeefcdea5a0 2024-11-24T20:48:05,941 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/c6a2364a8de440e49010a81578fca086 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/C/c6a2364a8de440e49010a81578fca086 2024-11-24T20:48:05,943 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/recovered.edits/442.seqid to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead/recovered.edits/442.seqid 2024-11-24T20:48:05,943 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/1bb3e8397087a4722761784332990ead 2024-11-24T20:48:05,944 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T20:48:05,945 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:48:05,947 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T20:48:05,949 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T20:48:05,950 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:48:05,950 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T20:48:05,950 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732481285950"}]},"ts":"9223372036854775807"} 2024-11-24T20:48:05,952 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T20:48:05,952 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1bb3e8397087a4722761784332990ead, NAME => 'TestAcidGuarantees,,1732481253161.1bb3e8397087a4722761784332990ead.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T20:48:05,952 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T20:48:05,952 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732481285952"}]},"ts":"9223372036854775807"} 2024-11-24T20:48:05,954 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T20:48:05,955 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:48:05,956 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 29 msec 2024-11-24T20:48:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-24T20:48:06,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T20:48:06,036 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:48:06,054 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=237 (was 239), OpenFileDescriptor=453 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=426 (was 522), ProcessCount=11 (was 11), AvailableMemoryMB=9254 (was 10011) 2024-11-24T20:48:06,064 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=237, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=426, ProcessCount=11, AvailableMemoryMB=9254 2024-11-24T20:48:06,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T20:48:06,066 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T20:48:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T20:48:06,069 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T20:48:06,069 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:06,069 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-11-24T20:48:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-24T20:48:06,070 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T20:48:06,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742127_1303 (size=959) 2024-11-24T20:48:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-24T20:48:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-24T20:48:06,480 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a 2024-11-24T20:48:06,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742128_1304 (size=53) 2024-11-24T20:48:06,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-24T20:48:06,890 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:48:06,890 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 72252cc9a6c51612da375fd8db66c697, disabling compactions & flushes 2024-11-24T20:48:06,890 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:06,890 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:06,890 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. after waiting 0 ms 2024-11-24T20:48:06,890 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:06,890 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:06,890 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 72252cc9a6c51612da375fd8db66c697: Waiting for close lock at 1732481286890Disabling compacts and flushes for region at 1732481286890Disabling writes for close at 1732481286890Writing region close event to WAL at 1732481286890Closed at 1732481286890 2024-11-24T20:48:06,891 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T20:48:06,892 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732481286891"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732481286891"}]},"ts":"1732481286891"} 2024-11-24T20:48:06,893 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T20:48:06,894 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T20:48:06,894 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732481286894"}]},"ts":"1732481286894"} 2024-11-24T20:48:06,895 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T20:48:06,896 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72252cc9a6c51612da375fd8db66c697, ASSIGN}] 2024-11-24T20:48:06,897 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72252cc9a6c51612da375fd8db66c697, ASSIGN 2024-11-24T20:48:06,897 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=72252cc9a6c51612da375fd8db66c697, ASSIGN; state=OFFLINE, location=20b981ef5c27,34625,1732481182291; forceNewPlan=false, retain=false 2024-11-24T20:48:07,048 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=72252cc9a6c51612da375fd8db66c697, regionState=OPENING, regionLocation=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:07,050 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=72252cc9a6c51612da375fd8db66c697, ASSIGN because future has completed 2024-11-24T20:48:07,050 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291}] 2024-11-24T20:48:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-24T20:48:07,205 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:07,205 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} 2024-11-24T20:48:07,205 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:07,205 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:48:07,205 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:07,205 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:07,207 INFO [StoreOpener-72252cc9a6c51612da375fd8db66c697-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:07,208 INFO [StoreOpener-72252cc9a6c51612da375fd8db66c697-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72252cc9a6c51612da375fd8db66c697 columnFamilyName A 2024-11-24T20:48:07,208 DEBUG [StoreOpener-72252cc9a6c51612da375fd8db66c697-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:07,208 INFO [StoreOpener-72252cc9a6c51612da375fd8db66c697-1 {}] regionserver.HStore(327): Store=72252cc9a6c51612da375fd8db66c697/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:48:07,208 INFO [StoreOpener-72252cc9a6c51612da375fd8db66c697-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:07,209 INFO [StoreOpener-72252cc9a6c51612da375fd8db66c697-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72252cc9a6c51612da375fd8db66c697 columnFamilyName B 2024-11-24T20:48:07,209 DEBUG [StoreOpener-72252cc9a6c51612da375fd8db66c697-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:07,209 INFO [StoreOpener-72252cc9a6c51612da375fd8db66c697-1 {}] regionserver.HStore(327): Store=72252cc9a6c51612da375fd8db66c697/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:48:07,209 INFO [StoreOpener-72252cc9a6c51612da375fd8db66c697-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:07,210 INFO [StoreOpener-72252cc9a6c51612da375fd8db66c697-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72252cc9a6c51612da375fd8db66c697 columnFamilyName C 2024-11-24T20:48:07,210 DEBUG [StoreOpener-72252cc9a6c51612da375fd8db66c697-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:07,211 INFO [StoreOpener-72252cc9a6c51612da375fd8db66c697-1 {}] regionserver.HStore(327): Store=72252cc9a6c51612da375fd8db66c697/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:48:07,211 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:07,211 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:07,212 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:07,212 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:07,212 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:07,213 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T20:48:07,214 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:07,215 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T20:48:07,215 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened 72252cc9a6c51612da375fd8db66c697; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72320297, jitterRate=0.07765640318393707}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T20:48:07,216 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:07,216 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for 72252cc9a6c51612da375fd8db66c697: Running coprocessor pre-open hook at 1732481287206Writing region info on filesystem at 1732481287206Initializing all the Stores at 1732481287206Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481287206Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481287206Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481287206Cleaning up temporary data from old regions at 1732481287212 (+6 ms)Running coprocessor post-open hooks at 1732481287216 (+4 ms)Region opened successfully at 1732481287216 2024-11-24T20:48:07,217 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., pid=89, masterSystemTime=1732481287202 2024-11-24T20:48:07,218 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:07,218 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:07,218 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=72252cc9a6c51612da375fd8db66c697, regionState=OPEN, openSeqNum=2, regionLocation=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:07,219 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 because future has completed 2024-11-24T20:48:07,222 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-24T20:48:07,222 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 in 170 msec 2024-11-24T20:48:07,223 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-24T20:48:07,223 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72252cc9a6c51612da375fd8db66c697, ASSIGN in 326 msec 2024-11-24T20:48:07,224 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T20:48:07,224 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732481287224"}]},"ts":"1732481287224"} 2024-11-24T20:48:07,225 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T20:48:07,226 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T20:48:07,227 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1600 sec 2024-11-24T20:48:08,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-24T20:48:08,215 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:48:08,216 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554bdeef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:08,216 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:48:08,216 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:48:08,217 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:48:08,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:48:08,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:48:08,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4864b342, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:08,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:48:08,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:48:08,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:08,219 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34672, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:48:08,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cde4dfe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:08,220 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:48:08,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:48:08,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:48:08,223 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:48:08,224 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T20:48:08,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 20b981ef5c27,38339,1732481181363 2024-11-24T20:48:08,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5aa3bc76 2024-11-24T20:48:08,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T20:48:08,226 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T20:48:08,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T20:48:08,229 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T20:48:08,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-24T20:48:08,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742129_1305 (size=995) 2024-11-24T20:48:08,644 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-24T20:48:08,644 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-24T20:48:08,649 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T20:48:08,652 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72252cc9a6c51612da375fd8db66c697, REOPEN/MOVE}] 2024-11-24T20:48:08,653 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72252cc9a6c51612da375fd8db66c697, REOPEN/MOVE 2024-11-24T20:48:08,654 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=72252cc9a6c51612da375fd8db66c697, regionState=CLOSING, regionLocation=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:08,655 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=72252cc9a6c51612da375fd8db66c697, REOPEN/MOVE because future has completed 2024-11-24T20:48:08,655 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T20:48:08,655 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291}] 2024-11-24T20:48:08,809 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:08,809 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T20:48:08,809 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing 72252cc9a6c51612da375fd8db66c697, disabling compactions & flushes 2024-11-24T20:48:08,809 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:08,809 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:08,809 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. after waiting 0 ms 2024-11-24T20:48:08,809 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:08,819 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T20:48:08,820 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:08,820 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for 72252cc9a6c51612da375fd8db66c697: Waiting for close lock at 1732481288809Running coprocessor pre-close hooks at 1732481288809Disabling compacts and flushes for region at 1732481288809Disabling writes for close at 1732481288809Writing region close event to WAL at 1732481288810 (+1 ms)Running coprocessor post-close hooks at 1732481288820 (+10 ms)Closed at 1732481288820 2024-11-24T20:48:08,820 WARN [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: 72252cc9a6c51612da375fd8db66c697 to self. 2024-11-24T20:48:08,822 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:08,823 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=72252cc9a6c51612da375fd8db66c697, regionState=CLOSED 2024-11-24T20:48:08,824 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 because future has completed 2024-11-24T20:48:08,828 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-24T20:48:08,828 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure 72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 in 170 msec 2024-11-24T20:48:08,829 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=72252cc9a6c51612da375fd8db66c697, REOPEN/MOVE; state=CLOSED, location=20b981ef5c27,34625,1732481182291; forceNewPlan=false, retain=true 2024-11-24T20:48:08,980 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=72252cc9a6c51612da375fd8db66c697, regionState=OPENING, regionLocation=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:08,984 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=72252cc9a6c51612da375fd8db66c697, REOPEN/MOVE because future has completed 2024-11-24T20:48:08,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291}] 2024-11-24T20:48:09,143 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:09,143 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} 2024-11-24T20:48:09,143 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:09,143 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:48:09,143 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:09,143 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:09,145 INFO [StoreOpener-72252cc9a6c51612da375fd8db66c697-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:09,146 INFO [StoreOpener-72252cc9a6c51612da375fd8db66c697-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72252cc9a6c51612da375fd8db66c697 columnFamilyName A 2024-11-24T20:48:09,147 DEBUG [StoreOpener-72252cc9a6c51612da375fd8db66c697-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:09,148 INFO [StoreOpener-72252cc9a6c51612da375fd8db66c697-1 {}] regionserver.HStore(327): Store=72252cc9a6c51612da375fd8db66c697/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:48:09,148 INFO [StoreOpener-72252cc9a6c51612da375fd8db66c697-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:09,149 INFO [StoreOpener-72252cc9a6c51612da375fd8db66c697-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72252cc9a6c51612da375fd8db66c697 columnFamilyName B 2024-11-24T20:48:09,149 DEBUG [StoreOpener-72252cc9a6c51612da375fd8db66c697-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:09,150 INFO [StoreOpener-72252cc9a6c51612da375fd8db66c697-1 {}] regionserver.HStore(327): Store=72252cc9a6c51612da375fd8db66c697/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:48:09,150 INFO [StoreOpener-72252cc9a6c51612da375fd8db66c697-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:09,151 INFO [StoreOpener-72252cc9a6c51612da375fd8db66c697-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72252cc9a6c51612da375fd8db66c697 columnFamilyName C 2024-11-24T20:48:09,151 DEBUG [StoreOpener-72252cc9a6c51612da375fd8db66c697-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:09,151 INFO [StoreOpener-72252cc9a6c51612da375fd8db66c697-1 {}] regionserver.HStore(327): Store=72252cc9a6c51612da375fd8db66c697/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:48:09,152 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:09,153 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:09,154 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:09,155 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:09,155 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:09,156 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T20:48:09,157 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:09,158 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened 72252cc9a6c51612da375fd8db66c697; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71832598, jitterRate=0.07038912177085876}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T20:48:09,158 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:09,159 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for 72252cc9a6c51612da375fd8db66c697: Running coprocessor pre-open hook at 1732481289144Writing region info on filesystem at 1732481289144Initializing all the Stores at 1732481289144Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481289145 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481289145Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481289145Cleaning up temporary data from old regions at 1732481289155 (+10 ms)Running coprocessor post-open hooks at 1732481289158 (+3 ms)Region opened successfully at 1732481289159 (+1 ms) 2024-11-24T20:48:09,159 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., pid=94, masterSystemTime=1732481289139 2024-11-24T20:48:09,161 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:09,161 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:09,162 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=72252cc9a6c51612da375fd8db66c697, regionState=OPEN, openSeqNum=5, regionLocation=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:09,163 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 because future has completed 2024-11-24T20:48:09,165 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-11-24T20:48:09,165 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 in 178 msec 2024-11-24T20:48:09,168 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-24T20:48:09,168 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72252cc9a6c51612da375fd8db66c697, REOPEN/MOVE in 513 msec 2024-11-24T20:48:09,171 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-24T20:48:09,171 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 520 msec 2024-11-24T20:48:09,174 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 942 msec 2024-11-24T20:48:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-24T20:48:09,176 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:48:09,176 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16d62a3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:09,176 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:48:09,177 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:48:09,178 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:48:09,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:48:09,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:48:09,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@327a2304, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:09,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:48:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:48:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:09,180 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34704, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:48:09,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@395cf315, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:09,181 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6513d18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:09,181 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:48:09,181 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:48:09,182 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:48:09,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:48:09,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:48:09,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@585e711d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:09,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:48:09,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:48:09,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:09,183 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34726, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:48:09,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2869e6d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:09,184 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@510f8e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:09,184 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:48:09,184 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:48:09,184 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:48:09,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:48:09,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:48:09,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@550ac72a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:09,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:48:09,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:48:09,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:09,186 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34744, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:48:09,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5583ce35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:09,187 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74436e44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:09,187 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:48:09,187 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:48:09,187 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:48:09,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:48:09,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:48:09,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4413022c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:09,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:48:09,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:48:09,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:09,188 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:48:09,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130abfd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:09,189 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15099ff4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:09,189 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:48:09,189 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:48:09,190 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:48:09,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:48:09,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:48:09,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33cda173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:09,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:48:09,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:48:09,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:09,191 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34768, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:48:09,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5af01daa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:09,192 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e8c61ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:09,192 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:48:09,192 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:48:09,193 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:48:09,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:48:09,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:48:09,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44d996dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:09,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:48:09,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:48:09,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:09,194 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:48:09,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40b7e131, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:09,195 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@147a1007, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:09,195 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:48:09,195 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:48:09,195 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:48:09,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:48:09,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:48:09,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ecb72f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:09,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:48:09,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:48:09,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:09,196 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:48:09,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c084a57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:09,197 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b4d201d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:09,197 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:48:09,197 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:48:09,198 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:48:09,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:48:09,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:48:09,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bc3b120, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:09,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:48:09,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:48:09,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:09,199 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:48:09,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@228cd45b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:09,200 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d2b45c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:09,200 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:48:09,200 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:48:09,201 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:48:09,201 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:48:09,201 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:48:09,201 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28f2a215, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:09,201 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:48:09,201 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:48:09,201 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:09,202 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34838, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:48:09,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1536163e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:09,204 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36e1d45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:09,204 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:48:09,204 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:48:09,205 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:48:09,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:48:09,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:48:09,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41af5e1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:09,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:48:09,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:48:09,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:09,207 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34860, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:48:09,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14c4d7d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:09,208 DEBUG [Thread-1417 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:48:09,208 DEBUG [Thread-1436 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:48:09,209 DEBUG [Thread-1425 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:48:09,209 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:48:09,209 DEBUG [Thread-1430 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:48:09,209 DEBUG [Thread-1419 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:48:09,209 DEBUG [Thread-1423 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:48:09,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:48:09,209 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:48:09,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:48:09,209 DEBUG [Thread-1421 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:48:09,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:48:09,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:48:09,210 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:48:09,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:48:09,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:48:09,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:48:09,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:48:09,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:48:09,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:48:09,211 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40426, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:48:09,211 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40438, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:48:09,212 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40454, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:48:09,212 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40462, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:48:09,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:48:09,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:48:09,212 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40478, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:48:09,212 DEBUG [Thread-1432 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:48:09,213 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40488, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:48:09,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:48:09,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:48:09,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5] 2024-11-24T20:48:09,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5] 2024-11-24T20:48:09,214 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40496, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:48:09,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5] 2024-11-24T20:48:09,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5] 2024-11-24T20:48:09,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5] 2024-11-24T20:48:09,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5] 2024-11-24T20:48:09,217 DEBUG [Thread-1434 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:48:09,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5] 2024-11-24T20:48:09,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5] 2024-11-24T20:48:09,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:48:09,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:48:09,219 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40500, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:48:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5] 2024-11-24T20:48:09,221 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:48:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-24T20:48:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-24T20:48:09,223 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:48:09,223 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:48:09,223 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:48:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:09,226 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:48:09,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72252cc9a6c51612da375fd8db66c697 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T20:48:09,248 DEBUG [Thread-1428 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:48:09,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:48:09,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:48:09,251 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40508, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:48:09,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5] 2024-11-24T20:48:09,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124a02b0b7c01664f1dafb8e1f3d6a4c544_72252cc9a6c51612da375fd8db66c697 is 50, key is test_row_0/A:col10/1732481289223/Put/seqid=0 2024-11-24T20:48:09,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:09,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481349255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:09,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:09,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481349257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:09,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:09,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:09,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:09,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481349259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:09,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:09,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481349260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:09,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:09,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:09,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481349261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:09,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:09,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742130_1306 (size=12154) 2024-11-24T20:48:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-24T20:48:09,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-24T20:48:09,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:09,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:09,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:09,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:09,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:09,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481349375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:09,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:09,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481349375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:09,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:09,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:09,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481349375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:09,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:09,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:09,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481349375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:09,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:09,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481349375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:09,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:09,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-24T20:48:09,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:09,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:09,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:09,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:09,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:09,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:09,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-24T20:48:09,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481349585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:09,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:09,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481349585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:09,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:09,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481349585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:09,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:09,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:09,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481349585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:09,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:09,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481349585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:09,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:09,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-24T20:48:09,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:09,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:09,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:09,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:09,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:09,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:09,685 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:09,688 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124a02b0b7c01664f1dafb8e1f3d6a4c544_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124a02b0b7c01664f1dafb8e1f3d6a4c544_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:09,689 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/218af5c4e3724f74b3b35a9707539a08, store: [table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:09,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/218af5c4e3724f74b3b35a9707539a08 is 175, key is test_row_0/A:col10/1732481289223/Put/seqid=0 2024-11-24T20:48:09,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742131_1307 (size=30955) 2024-11-24T20:48:09,693 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/218af5c4e3724f74b3b35a9707539a08 2024-11-24T20:48:09,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/1bb5c213775545359a73cef659115066 is 50, key is test_row_0/B:col10/1732481289223/Put/seqid=0 2024-11-24T20:48:09,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742132_1308 (size=12001) 2024-11-24T20:48:09,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-24T20:48:09,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:09,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:09,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:09,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:09,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:09,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:09,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-24T20:48:09,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:09,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481349895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:09,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:09,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:09,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481349895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:09,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:09,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:09,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481349895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:09,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:09,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:09,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481349895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:09,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:09,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:09,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481349895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:09,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:09,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-24T20:48:09,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:09,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:09,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:09,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:09,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:09,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:10,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/1bb5c213775545359a73cef659115066 2024-11-24T20:48:10,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-24T20:48:10,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:10,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:10,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:10,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:10,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:10,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:10,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/06ea0b9cb3a246b383c292420b5e9260 is 50, key is test_row_0/C:col10/1732481289223/Put/seqid=0 2024-11-24T20:48:10,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742133_1309 (size=12001) 2024-11-24T20:48:10,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/06ea0b9cb3a246b383c292420b5e9260 2024-11-24T20:48:10,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/218af5c4e3724f74b3b35a9707539a08 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/218af5c4e3724f74b3b35a9707539a08 2024-11-24T20:48:10,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/218af5c4e3724f74b3b35a9707539a08, entries=150, sequenceid=17, filesize=30.2 K 2024-11-24T20:48:10,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/1bb5c213775545359a73cef659115066 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/1bb5c213775545359a73cef659115066 2024-11-24T20:48:10,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/1bb5c213775545359a73cef659115066, entries=150, sequenceid=17, filesize=11.7 K 2024-11-24T20:48:10,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/06ea0b9cb3a246b383c292420b5e9260 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/06ea0b9cb3a246b383c292420b5e9260 2024-11-24T20:48:10,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/06ea0b9cb3a246b383c292420b5e9260, entries=150, sequenceid=17, filesize=11.7 K 2024-11-24T20:48:10,169 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 72252cc9a6c51612da375fd8db66c697 in 943ms, sequenceid=17, compaction requested=false 2024-11-24T20:48:10,170 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-24T20:48:10,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:10,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-24T20:48:10,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:10,293 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 72252cc9a6c51612da375fd8db66c697 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-24T20:48:10,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e14e521bab114a48b286e8642ae9d3ac_72252cc9a6c51612da375fd8db66c697 is 50, key is test_row_0/A:col10/1732481289258/Put/seqid=0 2024-11-24T20:48:10,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742134_1310 (size=12154) 2024-11-24T20:48:10,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:10,308 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e14e521bab114a48b286e8642ae9d3ac_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e14e521bab114a48b286e8642ae9d3ac_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:10,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/348e5f4f822842c4bebf805c2963952f, store: [table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:10,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/348e5f4f822842c4bebf805c2963952f is 175, key is test_row_0/A:col10/1732481289258/Put/seqid=0 2024-11-24T20:48:10,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742135_1311 (size=30955) 2024-11-24T20:48:10,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-24T20:48:10,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:10,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:10,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:10,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481350419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:10,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:10,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481350421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:10,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:10,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:10,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:10,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:10,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:10,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:10,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481350421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:10,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:10,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:10,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:10,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:10,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:10,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481350421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:10,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:10,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:10,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:10,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:10,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481350422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:10,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:10,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:10,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:10,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:10,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481350535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:10,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:10,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:10,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:10,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:10,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481350535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:10,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:10,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:10,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:10,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:10,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481350535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:10,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:10,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:10,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:10,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:10,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481350535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:10,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:10,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:10,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:10,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:10,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481350535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:10,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:10,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:10,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:10,730 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/348e5f4f822842c4bebf805c2963952f 2024-11-24T20:48:10,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/14d1396f99254d869c66d87ae0a56718 is 50, key is test_row_0/B:col10/1732481289258/Put/seqid=0 2024-11-24T20:48:10,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742136_1312 (size=12001) 2024-11-24T20:48:10,741 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/14d1396f99254d869c66d87ae0a56718 2024-11-24T20:48:10,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:10,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481350745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:10,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:10,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:10,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481350745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:10,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:10,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:10,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481350745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:10,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481350745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:10,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481350745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:10,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:10,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:10,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:10,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/7d45786e1b7a4b9ea64fbe660a6a6cd9 is 50, key is test_row_0/C:col10/1732481289258/Put/seqid=0 2024-11-24T20:48:10,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742137_1313 (size=12001) 2024-11-24T20:48:10,825 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T20:48:11,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481351055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:11,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481351056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:11,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:11,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481351056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:11,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:11,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:11,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:11,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481351056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:11,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:11,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481351056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:11,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:11,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:11,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:11,155 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/7d45786e1b7a4b9ea64fbe660a6a6cd9 2024-11-24T20:48:11,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/348e5f4f822842c4bebf805c2963952f as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/348e5f4f822842c4bebf805c2963952f 2024-11-24T20:48:11,174 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/348e5f4f822842c4bebf805c2963952f, entries=150, sequenceid=40, filesize=30.2 K 2024-11-24T20:48:11,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/14d1396f99254d869c66d87ae0a56718 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/14d1396f99254d869c66d87ae0a56718 2024-11-24T20:48:11,183 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/14d1396f99254d869c66d87ae0a56718, entries=150, sequenceid=40, filesize=11.7 K 2024-11-24T20:48:11,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/7d45786e1b7a4b9ea64fbe660a6a6cd9 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/7d45786e1b7a4b9ea64fbe660a6a6cd9 2024-11-24T20:48:11,193 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/7d45786e1b7a4b9ea64fbe660a6a6cd9, entries=150, sequenceid=40, filesize=11.7 K 2024-11-24T20:48:11,195 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 72252cc9a6c51612da375fd8db66c697 in 901ms, sequenceid=40, compaction requested=false 2024-11-24T20:48:11,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:11,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:11,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-24T20:48:11,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-24T20:48:11,211 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-24T20:48:11,211 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9760 sec 2024-11-24T20:48:11,213 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.9910 sec 2024-11-24T20:48:11,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-24T20:48:11,366 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:48:11,366 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:48:11,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-24T20:48:11,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-24T20:48:11,368 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:48:11,369 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:48:11,369 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:48:11,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-24T20:48:11,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-24T20:48:11,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:11,521 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 72252cc9a6c51612da375fd8db66c697 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-24T20:48:11,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112482b9d14110b6472d9c7d08566a66f590_72252cc9a6c51612da375fd8db66c697 is 50, key is test_row_0/A:col10/1732481290419/Put/seqid=0 2024-11-24T20:48:11,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742138_1314 (size=12154) 2024-11-24T20:48:11,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:11,534 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112482b9d14110b6472d9c7d08566a66f590_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112482b9d14110b6472d9c7d08566a66f590_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:11,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/462360228d564326a4ec4f5238e826f8, store: [table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:11,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/462360228d564326a4ec4f5238e826f8 is 175, key is test_row_0/A:col10/1732481290419/Put/seqid=0 2024-11-24T20:48:11,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742139_1315 (size=30955) 2024-11-24T20:48:11,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:11,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:11,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:11,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481351597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:11,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:11,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:11,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481351598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:11,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:11,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:11,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481351603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:11,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:11,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:11,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481351606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:11,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:11,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:11,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481351607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:11,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:11,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-24T20:48:11,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:11,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481351715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:11,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:11,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481351716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:11,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:11,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:11,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481351725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:11,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:11,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:11,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481351725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:11,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:11,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:11,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481351726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:11,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:11,940 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/462360228d564326a4ec4f5238e826f8 2024-11-24T20:48:11,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:11,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481351935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:11,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:11,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481351935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:11,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:11,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/3eb3bfbb35e34238801178f39f5526ab is 50, key is test_row_0/B:col10/1732481290419/Put/seqid=0 2024-11-24T20:48:11,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742140_1316 (size=12001) 2024-11-24T20:48:11,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481351945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:11,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:11,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:11,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481351945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:11,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:11,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:11,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481351946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:11,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:11,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:11,955 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/3eb3bfbb35e34238801178f39f5526ab 2024-11-24T20:48:11,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/0f76ec902c704659bcb167d8a36801f8 is 50, key is test_row_0/C:col10/1732481290419/Put/seqid=0 2024-11-24T20:48:11,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742141_1317 (size=12001) 2024-11-24T20:48:11,969 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/0f76ec902c704659bcb167d8a36801f8 2024-11-24T20:48:11,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/462360228d564326a4ec4f5238e826f8 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/462360228d564326a4ec4f5238e826f8 2024-11-24T20:48:11,979 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/462360228d564326a4ec4f5238e826f8, entries=150, sequenceid=53, filesize=30.2 K 2024-11-24T20:48:11,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/3eb3bfbb35e34238801178f39f5526ab as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/3eb3bfbb35e34238801178f39f5526ab 2024-11-24T20:48:11,989 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/3eb3bfbb35e34238801178f39f5526ab, entries=150, sequenceid=53, filesize=11.7 K 2024-11-24T20:48:11,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/0f76ec902c704659bcb167d8a36801f8 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/0f76ec902c704659bcb167d8a36801f8 2024-11-24T20:48:11,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-24T20:48:11,997 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/0f76ec902c704659bcb167d8a36801f8, entries=150, sequenceid=53, filesize=11.7 K 2024-11-24T20:48:11,998 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 72252cc9a6c51612da375fd8db66c697 in 476ms, sequenceid=53, compaction requested=true 2024-11-24T20:48:11,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:11,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:11,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-24T20:48:11,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-24T20:48:12,003 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-24T20:48:12,003 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 630 msec 2024-11-24T20:48:12,006 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 638 msec 2024-11-24T20:48:12,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:12,249 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:48:12,249 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72252cc9a6c51612da375fd8db66c697 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-24T20:48:12,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411246ff5ccd295964929b61164f7b289cd89_72252cc9a6c51612da375fd8db66c697 is 50, key is test_row_0/A:col10/1732481292249/Put/seqid=0 2024-11-24T20:48:12,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742142_1318 (size=17034) 2024-11-24T20:48:12,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:12,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481352267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:12,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:12,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481352270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:12,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:12,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:12,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481352270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:12,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:12,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:12,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481352271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:12,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:12,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:12,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481352277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:12,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:12,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481352385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:12,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481352385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:12,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481352395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:12,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:12,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481352395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:12,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:12,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481352396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:12,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:12,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-24T20:48:12,505 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:48:12,505 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:48:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-24T20:48:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T20:48:12,507 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:48:12,507 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:48:12,507 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:48:12,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481352595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:12,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481352595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:12,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481352604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:12,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:12,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481352605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:12,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:12,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:12,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481352605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:12,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:12,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:12,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T20:48:12,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T20:48:12,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:12,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:12,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:12,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:12,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:12,664 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:12,667 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411246ff5ccd295964929b61164f7b289cd89_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246ff5ccd295964929b61164f7b289cd89_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:12,667 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/0c7a178dc21d47c39ef39b2f5160b801, store: [table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:12,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/0c7a178dc21d47c39ef39b2f5160b801 is 175, key is test_row_0/A:col10/1732481292249/Put/seqid=0 2024-11-24T20:48:12,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742143_1319 (size=48139) 2024-11-24T20:48:12,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T20:48:12,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:12,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:12,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:12,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:12,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:12,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:12,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T20:48:12,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:12,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481352905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:12,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:12,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:12,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481352914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:12,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:12,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481352915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:12,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:12,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:12,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481352915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:12,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:12,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:12,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:12,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481352925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:12,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:12,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:12,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T20:48:12,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:12,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:12,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:12,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:12,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:12,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:13,072 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/0c7a178dc21d47c39ef39b2f5160b801 2024-11-24T20:48:13,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/0e3321453d1e43449def4800a95d2176 is 50, key is test_row_0/B:col10/1732481292249/Put/seqid=0 2024-11-24T20:48:13,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742144_1320 (size=12001) 2024-11-24T20:48:13,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T20:48:13,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:13,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:13,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:13,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:13,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:13,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:13,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T20:48:13,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T20:48:13,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:13,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:13,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:13,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:13,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:13,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:13,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T20:48:13,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:13,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:13,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:13,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:13,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:13,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:13,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:13,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481353425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:13,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:13,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:13,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:13,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:13,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481353435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:13,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:13,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:13,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481353435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:13,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:13,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:13,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:13,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:13,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:13,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:13,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481353435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:13,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:13,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:13,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:13,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:13,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481353444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:13,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/0e3321453d1e43449def4800a95d2176 2024-11-24T20:48:13,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/eb3d6c49a98e4f2bb2a5a02e9ce79318 is 50, key is test_row_0/C:col10/1732481292249/Put/seqid=0 2024-11-24T20:48:13,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742145_1321 (size=12001) 2024-11-24T20:48:13,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T20:48:13,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:13,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:13,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:13,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:13,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:13,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:13,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T20:48:13,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T20:48:13,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:13,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:13,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:13,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:13,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:13,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:13,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T20:48:13,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:13,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:13,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:13,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:13,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:13,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:13,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/eb3d6c49a98e4f2bb2a5a02e9ce79318 2024-11-24T20:48:13,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/0c7a178dc21d47c39ef39b2f5160b801 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/0c7a178dc21d47c39ef39b2f5160b801 2024-11-24T20:48:13,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/0c7a178dc21d47c39ef39b2f5160b801, entries=250, sequenceid=78, filesize=47.0 K 2024-11-24T20:48:13,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/0e3321453d1e43449def4800a95d2176 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/0e3321453d1e43449def4800a95d2176 2024-11-24T20:48:13,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/0e3321453d1e43449def4800a95d2176, entries=150, sequenceid=78, filesize=11.7 K 2024-11-24T20:48:13,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/eb3d6c49a98e4f2bb2a5a02e9ce79318 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/eb3d6c49a98e4f2bb2a5a02e9ce79318 2024-11-24T20:48:13,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/eb3d6c49a98e4f2bb2a5a02e9ce79318, entries=150, sequenceid=78, filesize=11.7 K 2024-11-24T20:48:13,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 72252cc9a6c51612da375fd8db66c697 in 1658ms, sequenceid=78, compaction requested=true 2024-11-24T20:48:13,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:13,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72252cc9a6c51612da375fd8db66c697:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:48:13,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:13,908 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:48:13,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72252cc9a6c51612da375fd8db66c697:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:48:13,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:13,908 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:48:13,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72252cc9a6c51612da375fd8db66c697:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:48:13,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:13,908 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:48:13,908 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 72252cc9a6c51612da375fd8db66c697/A is initiating minor compaction (all files) 2024-11-24T20:48:13,909 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72252cc9a6c51612da375fd8db66c697/A in TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:13,909 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/218af5c4e3724f74b3b35a9707539a08, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/348e5f4f822842c4bebf805c2963952f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/462360228d564326a4ec4f5238e826f8, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/0c7a178dc21d47c39ef39b2f5160b801] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp, totalSize=137.7 K 2024-11-24T20:48:13,909 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:13,909 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. files: [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/218af5c4e3724f74b3b35a9707539a08, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/348e5f4f822842c4bebf805c2963952f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/462360228d564326a4ec4f5238e826f8, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/0c7a178dc21d47c39ef39b2f5160b801] 2024-11-24T20:48:13,909 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:48:13,909 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 72252cc9a6c51612da375fd8db66c697/B is initiating minor compaction (all files) 2024-11-24T20:48:13,909 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72252cc9a6c51612da375fd8db66c697/B in TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:13,909 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 218af5c4e3724f74b3b35a9707539a08, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732481289223 2024-11-24T20:48:13,909 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/1bb5c213775545359a73cef659115066, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/14d1396f99254d869c66d87ae0a56718, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/3eb3bfbb35e34238801178f39f5526ab, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/0e3321453d1e43449def4800a95d2176] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp, totalSize=46.9 K 2024-11-24T20:48:13,909 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 348e5f4f822842c4bebf805c2963952f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732481289251 2024-11-24T20:48:13,909 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bb5c213775545359a73cef659115066, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732481289223 2024-11-24T20:48:13,909 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 462360228d564326a4ec4f5238e826f8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732481290407 2024-11-24T20:48:13,909 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 14d1396f99254d869c66d87ae0a56718, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732481289251 2024-11-24T20:48:13,910 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c7a178dc21d47c39ef39b2f5160b801, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732481291602 2024-11-24T20:48:13,910 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 3eb3bfbb35e34238801178f39f5526ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732481290407 2024-11-24T20:48:13,910 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e3321453d1e43449def4800a95d2176, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732481291605 2024-11-24T20:48:13,916 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:13,917 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72252cc9a6c51612da375fd8db66c697#B#compaction#275 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:13,918 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/295b194b00eb41fc89340ec209038fa6 is 50, key is test_row_0/B:col10/1732481292249/Put/seqid=0 2024-11-24T20:48:13,928 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124983e3b7471a04dd0be979b8f835821b2_72252cc9a6c51612da375fd8db66c697 store=[table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:13,930 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124983e3b7471a04dd0be979b8f835821b2_72252cc9a6c51612da375fd8db66c697, store=[table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:13,930 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124983e3b7471a04dd0be979b8f835821b2_72252cc9a6c51612da375fd8db66c697 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:13,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742146_1322 (size=12139) 2024-11-24T20:48:13,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742147_1323 (size=4469) 2024-11-24T20:48:14,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T20:48:14,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:14,034 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 72252cc9a6c51612da375fd8db66c697 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T20:48:14,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411249155b63017dd4f978da6e195f241a302_72252cc9a6c51612da375fd8db66c697 is 50, key is test_row_0/A:col10/1732481292276/Put/seqid=0 2024-11-24T20:48:14,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742148_1324 (size=12154) 2024-11-24T20:48:14,337 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/295b194b00eb41fc89340ec209038fa6 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/295b194b00eb41fc89340ec209038fa6 2024-11-24T20:48:14,338 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72252cc9a6c51612da375fd8db66c697#A#compaction#276 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:14,338 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/1317fe1c369145c38222d3cf44b258de is 175, key is test_row_0/A:col10/1732481292249/Put/seqid=0 2024-11-24T20:48:14,343 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 72252cc9a6c51612da375fd8db66c697/B of 72252cc9a6c51612da375fd8db66c697 into 295b194b00eb41fc89340ec209038fa6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:14,343 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:14,343 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., storeName=72252cc9a6c51612da375fd8db66c697/B, priority=12, startTime=1732481293908; duration=0sec 2024-11-24T20:48:14,343 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:14,343 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72252cc9a6c51612da375fd8db66c697:B 2024-11-24T20:48:14,343 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:48:14,344 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:48:14,345 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 72252cc9a6c51612da375fd8db66c697/C is initiating minor compaction (all files) 2024-11-24T20:48:14,345 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72252cc9a6c51612da375fd8db66c697/C in TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:14,345 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/06ea0b9cb3a246b383c292420b5e9260, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/7d45786e1b7a4b9ea64fbe660a6a6cd9, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/0f76ec902c704659bcb167d8a36801f8, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/eb3d6c49a98e4f2bb2a5a02e9ce79318] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp, totalSize=46.9 K 2024-11-24T20:48:14,345 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 06ea0b9cb3a246b383c292420b5e9260, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732481289223 2024-11-24T20:48:14,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742149_1325 (size=31093) 2024-11-24T20:48:14,345 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d45786e1b7a4b9ea64fbe660a6a6cd9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732481289251 2024-11-24T20:48:14,346 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f76ec902c704659bcb167d8a36801f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732481290407 2024-11-24T20:48:14,346 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting eb3d6c49a98e4f2bb2a5a02e9ce79318, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732481291605 2024-11-24T20:48:14,354 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/1317fe1c369145c38222d3cf44b258de as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/1317fe1c369145c38222d3cf44b258de 2024-11-24T20:48:14,359 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 72252cc9a6c51612da375fd8db66c697/A of 72252cc9a6c51612da375fd8db66c697 into 1317fe1c369145c38222d3cf44b258de(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:14,359 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:14,359 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., storeName=72252cc9a6c51612da375fd8db66c697/A, priority=12, startTime=1732481293907; duration=0sec 2024-11-24T20:48:14,359 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:14,359 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72252cc9a6c51612da375fd8db66c697:A 2024-11-24T20:48:14,364 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72252cc9a6c51612da375fd8db66c697#C#compaction#278 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:14,365 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/27732c7e3d4346e29935d08c3d6d1ff7 is 50, key is test_row_0/C:col10/1732481292249/Put/seqid=0 2024-11-24T20:48:14,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742150_1326 (size=12139) 2024-11-24T20:48:14,382 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/27732c7e3d4346e29935d08c3d6d1ff7 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/27732c7e3d4346e29935d08c3d6d1ff7 2024-11-24T20:48:14,386 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 72252cc9a6c51612da375fd8db66c697/C of 72252cc9a6c51612da375fd8db66c697 into 27732c7e3d4346e29935d08c3d6d1ff7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:14,386 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:14,386 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., storeName=72252cc9a6c51612da375fd8db66c697/C, priority=12, startTime=1732481293908; duration=0sec 2024-11-24T20:48:14,387 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:14,387 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72252cc9a6c51612da375fd8db66c697:C 2024-11-24T20:48:14,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:14,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:14,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:14,447 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411249155b63017dd4f978da6e195f241a302_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249155b63017dd4f978da6e195f241a302_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:14,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/e045dd7cc07f4bb39f543be5d9c56ee4, store: [table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:14,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/e045dd7cc07f4bb39f543be5d9c56ee4 is 175, key is test_row_0/A:col10/1732481292276/Put/seqid=0 2024-11-24T20:48:14,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742151_1327 (size=30955) 2024-11-24T20:48:14,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:14,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481354482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:14,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:14,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:14,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:14,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:14,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481354482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:14,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:14,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:14,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:14,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:14,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481354486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:14,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:14,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:14,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:14,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:14,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481354487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:14,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:14,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481354490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:14,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:14,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481354594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:14,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:14,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:14,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:14,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:14,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481354595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:14,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:14,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:14,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:14,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:14,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481354595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:14,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:14,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:14,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:14,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:14,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481354605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:14,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:14,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481354605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:14,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T20:48:14,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481354805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:14,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481354805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:14,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:14,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:14,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481354805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:14,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:14,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481354815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:14,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:14,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:14,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:14,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:14,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481354815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:14,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:14,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:14,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:14,852 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=89, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/e045dd7cc07f4bb39f543be5d9c56ee4 2024-11-24T20:48:14,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/9d0d7aed725c43be975986ca948b95ca is 50, key is test_row_0/B:col10/1732481292276/Put/seqid=0 2024-11-24T20:48:14,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742152_1328 (size=12001) 2024-11-24T20:48:15,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481355115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:15,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:15,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:15,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:15,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481355115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:15,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:15,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:15,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:15,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481355115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:15,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:15,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:15,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:15,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:15,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481355125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:15,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:15,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:15,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:15,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481355125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:15,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:15,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:15,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:15,262 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/9d0d7aed725c43be975986ca948b95ca 2024-11-24T20:48:15,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/a85aa1a9680c4ad78bbcfd335f1a69d1 is 50, key is test_row_0/C:col10/1732481292276/Put/seqid=0 2024-11-24T20:48:15,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742153_1329 (size=12001) 2024-11-24T20:48:15,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:15,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481355625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:15,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:15,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:15,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:15,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:15,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481355625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:15,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:15,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:15,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:15,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481355635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:15,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:15,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:15,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:15,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:15,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481355645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:15,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:15,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:15,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:15,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:15,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481355645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:15,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:15,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:15,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:15,677 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/a85aa1a9680c4ad78bbcfd335f1a69d1 2024-11-24T20:48:15,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/e045dd7cc07f4bb39f543be5d9c56ee4 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/e045dd7cc07f4bb39f543be5d9c56ee4 2024-11-24T20:48:15,685 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/e045dd7cc07f4bb39f543be5d9c56ee4, entries=150, sequenceid=89, filesize=30.2 K 2024-11-24T20:48:15,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/9d0d7aed725c43be975986ca948b95ca as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/9d0d7aed725c43be975986ca948b95ca 2024-11-24T20:48:15,689 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/9d0d7aed725c43be975986ca948b95ca, entries=150, sequenceid=89, filesize=11.7 K 2024-11-24T20:48:15,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/a85aa1a9680c4ad78bbcfd335f1a69d1 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/a85aa1a9680c4ad78bbcfd335f1a69d1 2024-11-24T20:48:15,693 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/a85aa1a9680c4ad78bbcfd335f1a69d1, entries=150, sequenceid=89, filesize=11.7 K 2024-11-24T20:48:15,694 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 72252cc9a6c51612da375fd8db66c697 in 1660ms, sequenceid=89, compaction requested=false 2024-11-24T20:48:15,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:15,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:15,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-24T20:48:15,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-24T20:48:15,697 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-24T20:48:15,697 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1890 sec 2024-11-24T20:48:15,698 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 3.1920 sec 2024-11-24T20:48:16,640 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:48:16,640 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72252cc9a6c51612da375fd8db66c697 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-24T20:48:16,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:16,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411247439300b46aa48d39337c2b4054ef47d_72252cc9a6c51612da375fd8db66c697 is 50, key is test_row_0/A:col10/1732481294481/Put/seqid=0 2024-11-24T20:48:16,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742154_1330 (size=14594) 2024-11-24T20:48:16,658 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:16,662 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411247439300b46aa48d39337c2b4054ef47d_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247439300b46aa48d39337c2b4054ef47d_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:16,663 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/068f10e08df941d191983c7e04648326, store: [table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:16,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/068f10e08df941d191983c7e04648326 is 175, key is test_row_0/A:col10/1732481294481/Put/seqid=0 2024-11-24T20:48:16,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742155_1331 (size=39549) 2024-11-24T20:48:16,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:16,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481356662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:16,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:16,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:16,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:16,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:16,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481356663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:16,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:16,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:16,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:16,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:16,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481356665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:16,675 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_2' in 72252cc9a6c51612da375fd8db66c697 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:48:16,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:16,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:16,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:16,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T20:48:16,679 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:48:16,680 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:48:16,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-24T20:48:16,681 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:48:16,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T20:48:16,682 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:48:16,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:16,682 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:48:16,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481356669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:16,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:16,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481356669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:16,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:16,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:16,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:16,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:16,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:16,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:16,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T20:48:16,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:16,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481356784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:16,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:16,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:16,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:16,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481356785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:16,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:16,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:16,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:16,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:16,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:16,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481356785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:16,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:16,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481356785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:16,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:16,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:16,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:16,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:16,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:16,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:16,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T20:48:16,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:16,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:16,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:16,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:16,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:16,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:16,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T20:48:16,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:16,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:16,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:16,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:16,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:16,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T20:48:17,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:17,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481356995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:17,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:17,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481356995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:17,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:17,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:17,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:17,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:17,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:17,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481356995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:17,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:17,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:17,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:17,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:17,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:17,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481356995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:17,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:17,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:17,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:17,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:17,074 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/068f10e08df941d191983c7e04648326 2024-11-24T20:48:17,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/60f581b50eb343cdbfb8c5d8cb949414 is 50, key is test_row_0/B:col10/1732481294481/Put/seqid=0 2024-11-24T20:48:17,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742156_1332 (size=12001) 2024-11-24T20:48:17,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T20:48:17,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:17,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:17,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:17,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:17,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:17,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T20:48:17,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:17,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:17,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:17,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:17,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:17,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:17,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T20:48:17,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:17,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481357305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:17,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:17,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481357305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:17,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:17,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481357305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:17,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:17,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481357315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:17,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:17,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:17,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:17,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T20:48:17,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:17,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:17,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:17,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:17,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:17,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:17,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/60f581b50eb343cdbfb8c5d8cb949414 2024-11-24T20:48:17,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/6965fdc7e71a497c8efc5f46f7196113 is 50, key is test_row_0/C:col10/1732481294481/Put/seqid=0 2024-11-24T20:48:17,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742157_1333 (size=12001) 2024-11-24T20:48:17,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/6965fdc7e71a497c8efc5f46f7196113 2024-11-24T20:48:17,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/068f10e08df941d191983c7e04648326 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/068f10e08df941d191983c7e04648326 2024-11-24T20:48:17,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/068f10e08df941d191983c7e04648326, entries=200, sequenceid=118, filesize=38.6 K 2024-11-24T20:48:17,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/60f581b50eb343cdbfb8c5d8cb949414 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/60f581b50eb343cdbfb8c5d8cb949414 2024-11-24T20:48:17,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/60f581b50eb343cdbfb8c5d8cb949414, entries=150, sequenceid=118, filesize=11.7 K 2024-11-24T20:48:17,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/6965fdc7e71a497c8efc5f46f7196113 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/6965fdc7e71a497c8efc5f46f7196113 2024-11-24T20:48:17,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/6965fdc7e71a497c8efc5f46f7196113, entries=150, sequenceid=118, filesize=11.7 K 2024-11-24T20:48:17,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 72252cc9a6c51612da375fd8db66c697 in 894ms, sequenceid=118, compaction requested=true 2024-11-24T20:48:17,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:17,534 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:17,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72252cc9a6c51612da375fd8db66c697:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:48:17,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:17,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72252cc9a6c51612da375fd8db66c697:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:48:17,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:17,534 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:17,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72252cc9a6c51612da375fd8db66c697:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:48:17,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:17,535 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101597 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:17,535 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 72252cc9a6c51612da375fd8db66c697/A is initiating minor compaction (all files) 2024-11-24T20:48:17,536 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72252cc9a6c51612da375fd8db66c697/A in TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:17,536 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/1317fe1c369145c38222d3cf44b258de, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/e045dd7cc07f4bb39f543be5d9c56ee4, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/068f10e08df941d191983c7e04648326] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp, totalSize=99.2 K 2024-11-24T20:48:17,536 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:17,536 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. files: [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/1317fe1c369145c38222d3cf44b258de, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/e045dd7cc07f4bb39f543be5d9c56ee4, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/068f10e08df941d191983c7e04648326] 2024-11-24T20:48:17,536 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:17,536 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 72252cc9a6c51612da375fd8db66c697/B is initiating minor compaction (all files) 2024-11-24T20:48:17,536 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72252cc9a6c51612da375fd8db66c697/B in TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:17,536 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/295b194b00eb41fc89340ec209038fa6, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/9d0d7aed725c43be975986ca948b95ca, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/60f581b50eb343cdbfb8c5d8cb949414] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp, totalSize=35.3 K 2024-11-24T20:48:17,536 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1317fe1c369145c38222d3cf44b258de, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732481291605 2024-11-24T20:48:17,537 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 295b194b00eb41fc89340ec209038fa6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732481291605 2024-11-24T20:48:17,537 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting e045dd7cc07f4bb39f543be5d9c56ee4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732481292266 2024-11-24T20:48:17,537 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d0d7aed725c43be975986ca948b95ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732481292266 2024-11-24T20:48:17,537 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 60f581b50eb343cdbfb8c5d8cb949414, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732481294479 2024-11-24T20:48:17,537 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 068f10e08df941d191983c7e04648326, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732481294479 2024-11-24T20:48:17,554 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72252cc9a6c51612da375fd8db66c697#B#compaction#284 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:17,555 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/6e7bb6b8e7344f0088d5764c3218b7ea is 50, key is test_row_0/B:col10/1732481294481/Put/seqid=0 2024-11-24T20:48:17,561 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:17,580 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411246ac4bed2761f49f6b99ad6a0896afdfa_72252cc9a6c51612da375fd8db66c697 store=[table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:17,581 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411246ac4bed2761f49f6b99ad6a0896afdfa_72252cc9a6c51612da375fd8db66c697, store=[table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:17,582 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411246ac4bed2761f49f6b99ad6a0896afdfa_72252cc9a6c51612da375fd8db66c697 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:17,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742158_1334 (size=12241) 2024-11-24T20:48:17,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742159_1335 (size=4469) 2024-11-24T20:48:17,596 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/6e7bb6b8e7344f0088d5764c3218b7ea as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/6e7bb6b8e7344f0088d5764c3218b7ea 2024-11-24T20:48:17,597 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72252cc9a6c51612da375fd8db66c697#A#compaction#285 average throughput is 0.68 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:17,597 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/6f675e6880314028aeba0f18b346d3fc is 175, key is test_row_0/A:col10/1732481294481/Put/seqid=0 2024-11-24T20:48:17,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T20:48:17,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:17,602 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72252cc9a6c51612da375fd8db66c697/B of 72252cc9a6c51612da375fd8db66c697 into 6e7bb6b8e7344f0088d5764c3218b7ea(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:17,602 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 72252cc9a6c51612da375fd8db66c697 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T20:48:17,602 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:17,602 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., storeName=72252cc9a6c51612da375fd8db66c697/B, priority=13, startTime=1732481297534; duration=0sec 2024-11-24T20:48:17,602 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:17,602 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72252cc9a6c51612da375fd8db66c697:B 2024-11-24T20:48:17,603 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:17,605 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:17,605 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 72252cc9a6c51612da375fd8db66c697/C is initiating minor compaction (all files) 2024-11-24T20:48:17,605 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72252cc9a6c51612da375fd8db66c697/C in TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:17,605 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/27732c7e3d4346e29935d08c3d6d1ff7, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/a85aa1a9680c4ad78bbcfd335f1a69d1, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/6965fdc7e71a497c8efc5f46f7196113] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp, totalSize=35.3 K 2024-11-24T20:48:17,606 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 27732c7e3d4346e29935d08c3d6d1ff7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732481291605 2024-11-24T20:48:17,609 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting a85aa1a9680c4ad78bbcfd335f1a69d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732481292266 2024-11-24T20:48:17,609 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 6965fdc7e71a497c8efc5f46f7196113, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732481294479 2024-11-24T20:48:17,623 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72252cc9a6c51612da375fd8db66c697#C#compaction#286 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:17,624 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/9739328e78ef4af09f13384b352ed71a is 50, key is test_row_0/C:col10/1732481294481/Put/seqid=0 2024-11-24T20:48:17,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742160_1336 (size=31195) 2024-11-24T20:48:17,630 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/6f675e6880314028aeba0f18b346d3fc as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/6f675e6880314028aeba0f18b346d3fc 2024-11-24T20:48:17,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243f597db534134416878b962dbf3a78ef_72252cc9a6c51612da375fd8db66c697 is 50, key is test_row_0/A:col10/1732481296660/Put/seqid=0 2024-11-24T20:48:17,635 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72252cc9a6c51612da375fd8db66c697/A of 72252cc9a6c51612da375fd8db66c697 into 6f675e6880314028aeba0f18b346d3fc(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:17,635 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:17,636 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., storeName=72252cc9a6c51612da375fd8db66c697/A, priority=13, startTime=1732481297534; duration=0sec 2024-11-24T20:48:17,636 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:17,636 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72252cc9a6c51612da375fd8db66c697:A 2024-11-24T20:48:17,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742161_1337 (size=12241) 2024-11-24T20:48:17,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742162_1338 (size=12154) 2024-11-24T20:48:17,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,652 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243f597db534134416878b962dbf3a78ef_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243f597db534134416878b962dbf3a78ef_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:17,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/8d4ac2e8bec14543a8079c7390ce1290, store: [table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:17,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/8d4ac2e8bec14543a8079c7390ce1290 is 175, key is test_row_0/A:col10/1732481296660/Put/seqid=0 2024-11-24T20:48:17,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742163_1339 (size=30955) 2024-11-24T20:48:17,662 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/8d4ac2e8bec14543a8079c7390ce1290 2024-11-24T20:48:17,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/cd22368b33c84fc6b39ae92347db7e3b is 50, key is test_row_0/B:col10/1732481296660/Put/seqid=0 2024-11-24T20:48:17,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742164_1340 (size=12001) 2024-11-24T20:48:17,671 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/cd22368b33c84fc6b39ae92347db7e3b 2024-11-24T20:48:17,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/968672f540e24e50a799506ed3e5791b is 50, key is test_row_0/C:col10/1732481296660/Put/seqid=0 2024-11-24T20:48:17,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742165_1341 (size=12001) 2024-11-24T20:48:17,684 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/968672f540e24e50a799506ed3e5791b 2024-11-24T20:48:17,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/8d4ac2e8bec14543a8079c7390ce1290 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/8d4ac2e8bec14543a8079c7390ce1290 2024-11-24T20:48:17,691 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/8d4ac2e8bec14543a8079c7390ce1290, entries=150, sequenceid=130, filesize=30.2 K 2024-11-24T20:48:17,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/cd22368b33c84fc6b39ae92347db7e3b as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/cd22368b33c84fc6b39ae92347db7e3b 2024-11-24T20:48:17,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,696 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/cd22368b33c84fc6b39ae92347db7e3b, entries=150, sequenceid=130, filesize=11.7 K 2024-11-24T20:48:17,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/968672f540e24e50a799506ed3e5791b as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/968672f540e24e50a799506ed3e5791b 2024-11-24T20:48:17,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,700 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/968672f540e24e50a799506ed3e5791b, entries=150, sequenceid=130, filesize=11.7 K 2024-11-24T20:48:17,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,701 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for 72252cc9a6c51612da375fd8db66c697 in 98ms, sequenceid=130, compaction requested=false 2024-11-24T20:48:17,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:17,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:17,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-24T20:48:17,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-24T20:48:17,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,704 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-24T20:48:17,704 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0200 sec 2024-11-24T20:48:17,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,705 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.0240 sec 2024-11-24T20:48:17,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T20:48:17,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,816 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:48:17,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,816 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:48:17,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-24T20:48:17,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,819 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:48:17,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T20:48:17,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,820 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:48:17,821 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:48:17,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:17,919 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:48:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72252cc9a6c51612da375fd8db66c697 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T20:48:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T20:48:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b9c564fa304c4038bf7036d603c662a9_72252cc9a6c51612da375fd8db66c697 is 50, key is test_row_0/A:col10/1732481297897/Put/seqid=0 2024-11-24T20:48:17,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742167_1343 (size=24758) 2024-11-24T20:48:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:17,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T20:48:17,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:17,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:17,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:17,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:17,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:17,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:17,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:17,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481357982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:17,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:17,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:17,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:18,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:18,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481357992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:18,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:18,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481357992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:18,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:18,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:18,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:18,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:18,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:18,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:18,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:18,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481357993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:18,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:18,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:18,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:18,044 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/9739328e78ef4af09f13384b352ed71a as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/9739328e78ef4af09f13384b352ed71a 2024-11-24T20:48:18,048 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72252cc9a6c51612da375fd8db66c697/C of 72252cc9a6c51612da375fd8db66c697 into 9739328e78ef4af09f13384b352ed71a(size=12.0 K), total size for store is 23.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:18,048 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:18,048 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., storeName=72252cc9a6c51612da375fd8db66c697/C, priority=13, startTime=1732481297534; duration=0sec 2024-11-24T20:48:18,048 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:18,048 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72252cc9a6c51612da375fd8db66c697:C 2024-11-24T20:48:18,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481358105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:18,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:18,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:18,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:18,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:18,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481358115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:18,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:18,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:18,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:18,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481358115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:18,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:18,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:18,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:18,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:18,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481358115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:18,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T20:48:18,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:18,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:18,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:18,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:18,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:18,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:18,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:18,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:18,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:18,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T20:48:18,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T20:48:18,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:18,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:18,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:18,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:18,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:18,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481358325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:18,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:18,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:18,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:18,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:18,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481358335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:18,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:18,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481358336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:18,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:18,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:18,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481358336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:18,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:18,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:18,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:18,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:18,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:18,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:18,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:18,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:18,363 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:18,374 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b9c564fa304c4038bf7036d603c662a9_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b9c564fa304c4038bf7036d603c662a9_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:18,376 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/df2b580779014fb1b67e16074a8fa46a, store: [table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:18,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/df2b580779014fb1b67e16074a8fa46a is 175, key is test_row_0/A:col10/1732481297897/Put/seqid=0 2024-11-24T20:48:18,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742166_1342 (size=74395) 2024-11-24T20:48:18,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T20:48:18,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:18,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:18,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:18,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:18,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:18,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:18,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T20:48:18,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T20:48:18,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:18,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:18,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:18,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:18,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:18,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:18,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481358645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:18,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:18,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481358655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:18,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:18,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:18,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:18,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481358655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:18,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:18,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481358655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:18,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:18,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481358695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:18,702 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_2' in 72252cc9a6c51612da375fd8db66c697 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:48:18,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:18,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:18,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:18,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T20:48:18,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:18,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:18,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:18,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:18,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:18,781 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=142, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/df2b580779014fb1b67e16074a8fa46a 2024-11-24T20:48:18,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/fb2fc177dda54cbca972de7ce4f74488 is 50, key is test_row_0/B:col10/1732481297897/Put/seqid=0 2024-11-24T20:48:18,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742168_1344 (size=12151) 2024-11-24T20:48:18,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T20:48:18,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:18,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:18,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:18,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:18,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T20:48:19,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T20:48:19,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:19,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:19,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:19,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:19,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:19,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:19,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481359155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:19,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:19,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:19,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:19,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:19,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481359165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:19,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:19,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:19,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:19,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:19,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481359176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:19,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:19,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481359176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:19,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:19,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:19,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:19,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:19,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:19,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:19,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T20:48:19,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:19,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:19,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:19,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:19,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:19,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:19,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/fb2fc177dda54cbca972de7ce4f74488 2024-11-24T20:48:19,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/5fbb96e3083e46e888da91ae39a484ac is 50, key is test_row_0/C:col10/1732481297897/Put/seqid=0 2024-11-24T20:48:19,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742169_1345 (size=12151) 2024-11-24T20:48:19,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/5fbb96e3083e46e888da91ae39a484ac 2024-11-24T20:48:19,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/df2b580779014fb1b67e16074a8fa46a as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/df2b580779014fb1b67e16074a8fa46a 2024-11-24T20:48:19,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/df2b580779014fb1b67e16074a8fa46a, entries=400, sequenceid=142, filesize=72.7 K 2024-11-24T20:48:19,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/fb2fc177dda54cbca972de7ce4f74488 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/fb2fc177dda54cbca972de7ce4f74488 2024-11-24T20:48:19,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/fb2fc177dda54cbca972de7ce4f74488, entries=150, sequenceid=142, filesize=11.9 K 2024-11-24T20:48:19,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/5fbb96e3083e46e888da91ae39a484ac as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/5fbb96e3083e46e888da91ae39a484ac 2024-11-24T20:48:19,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/5fbb96e3083e46e888da91ae39a484ac, entries=150, sequenceid=142, filesize=11.9 K 2024-11-24T20:48:19,323 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 72252cc9a6c51612da375fd8db66c697 in 1403ms, sequenceid=142, compaction requested=true 2024-11-24T20:48:19,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:19,323 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:19,324 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 136545 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:19,324 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 72252cc9a6c51612da375fd8db66c697/A is initiating minor compaction (all files) 2024-11-24T20:48:19,324 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72252cc9a6c51612da375fd8db66c697/A in TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:19,324 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/6f675e6880314028aeba0f18b346d3fc, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/8d4ac2e8bec14543a8079c7390ce1290, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/df2b580779014fb1b67e16074a8fa46a] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp, totalSize=133.3 K 2024-11-24T20:48:19,324 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:19,324 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. files: [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/6f675e6880314028aeba0f18b346d3fc, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/8d4ac2e8bec14543a8079c7390ce1290, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/df2b580779014fb1b67e16074a8fa46a] 2024-11-24T20:48:19,325 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f675e6880314028aeba0f18b346d3fc, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732481294479 2024-11-24T20:48:19,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72252cc9a6c51612da375fd8db66c697:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:48:19,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:19,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72252cc9a6c51612da375fd8db66c697:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:48:19,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:19,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72252cc9a6c51612da375fd8db66c697:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:48:19,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:19,325 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:19,328 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d4ac2e8bec14543a8079c7390ce1290, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732481296660 2024-11-24T20:48:19,328 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting df2b580779014fb1b67e16074a8fa46a, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732481297834 2024-11-24T20:48:19,329 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:19,329 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 72252cc9a6c51612da375fd8db66c697/B is initiating minor compaction (all files) 2024-11-24T20:48:19,329 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72252cc9a6c51612da375fd8db66c697/B in TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:19,329 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/6e7bb6b8e7344f0088d5764c3218b7ea, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/cd22368b33c84fc6b39ae92347db7e3b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/fb2fc177dda54cbca972de7ce4f74488] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp, totalSize=35.5 K 2024-11-24T20:48:19,330 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e7bb6b8e7344f0088d5764c3218b7ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732481294479 2024-11-24T20:48:19,333 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting cd22368b33c84fc6b39ae92347db7e3b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732481296660 2024-11-24T20:48:19,334 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting fb2fc177dda54cbca972de7ce4f74488, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732481297897 2024-11-24T20:48:19,351 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:19,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T20:48:19,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:19,355 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 72252cc9a6c51612da375fd8db66c697 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-24T20:48:19,362 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72252cc9a6c51612da375fd8db66c697#B#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:19,363 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/3c3e248dd0c94e83ad630e94eec28aa8 is 50, key is test_row_0/B:col10/1732481297897/Put/seqid=0 2024-11-24T20:48:19,375 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124d650a75cc25940ae930cb45b5edc8001_72252cc9a6c51612da375fd8db66c697 store=[table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:19,378 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124d650a75cc25940ae930cb45b5edc8001_72252cc9a6c51612da375fd8db66c697, store=[table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:19,379 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d650a75cc25940ae930cb45b5edc8001_72252cc9a6c51612da375fd8db66c697 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:19,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742170_1346 (size=12493) 2024-11-24T20:48:19,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112400b712bd490f4bc3b24b7eb4e21311f5_72252cc9a6c51612da375fd8db66c697 is 50, key is test_row_0/A:col10/1732481297981/Put/seqid=0 2024-11-24T20:48:19,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742171_1347 (size=4469) 2024-11-24T20:48:19,485 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72252cc9a6c51612da375fd8db66c697#A#compaction#293 average throughput is 0.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:19,486 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/479fd04a93054a4ba74e2d8ec98ea5ab is 175, key is test_row_0/A:col10/1732481297897/Put/seqid=0 2024-11-24T20:48:19,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742172_1348 (size=12304) 2024-11-24T20:48:19,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742173_1349 (size=31447) 2024-11-24T20:48:19,508 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/479fd04a93054a4ba74e2d8ec98ea5ab as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/479fd04a93054a4ba74e2d8ec98ea5ab 2024-11-24T20:48:19,515 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72252cc9a6c51612da375fd8db66c697/A of 72252cc9a6c51612da375fd8db66c697 into 479fd04a93054a4ba74e2d8ec98ea5ab(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:19,515 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:19,515 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., storeName=72252cc9a6c51612da375fd8db66c697/A, priority=13, startTime=1732481299323; duration=0sec 2024-11-24T20:48:19,515 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:19,515 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72252cc9a6c51612da375fd8db66c697:A 2024-11-24T20:48:19,515 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:19,517 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:19,517 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 72252cc9a6c51612da375fd8db66c697/C is initiating minor compaction (all files) 2024-11-24T20:48:19,517 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72252cc9a6c51612da375fd8db66c697/C in TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:19,517 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/9739328e78ef4af09f13384b352ed71a, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/968672f540e24e50a799506ed3e5791b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/5fbb96e3083e46e888da91ae39a484ac] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp, totalSize=35.5 K 2024-11-24T20:48:19,517 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9739328e78ef4af09f13384b352ed71a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732481294479 2024-11-24T20:48:19,518 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 968672f540e24e50a799506ed3e5791b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732481296660 2024-11-24T20:48:19,518 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fbb96e3083e46e888da91ae39a484ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732481297897 2024-11-24T20:48:19,533 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72252cc9a6c51612da375fd8db66c697#C#compaction#296 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:19,534 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/080ac9cf3a764fa6a904be634bb7f81f is 50, key is test_row_0/C:col10/1732481297897/Put/seqid=0 2024-11-24T20:48:19,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742174_1350 (size=12493) 2024-11-24T20:48:19,573 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/080ac9cf3a764fa6a904be634bb7f81f as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/080ac9cf3a764fa6a904be634bb7f81f 2024-11-24T20:48:19,579 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72252cc9a6c51612da375fd8db66c697/C of 72252cc9a6c51612da375fd8db66c697 into 080ac9cf3a764fa6a904be634bb7f81f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:19,579 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:19,580 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., storeName=72252cc9a6c51612da375fd8db66c697/C, priority=13, startTime=1732481299325; duration=0sec 2024-11-24T20:48:19,580 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:19,580 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72252cc9a6c51612da375fd8db66c697:C 2024-11-24T20:48:19,848 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/3c3e248dd0c94e83ad630e94eec28aa8 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/3c3e248dd0c94e83ad630e94eec28aa8 2024-11-24T20:48:19,858 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72252cc9a6c51612da375fd8db66c697/B of 72252cc9a6c51612da375fd8db66c697 into 3c3e248dd0c94e83ad630e94eec28aa8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:19,858 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:19,858 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., storeName=72252cc9a6c51612da375fd8db66c697/B, priority=13, startTime=1732481299325; duration=0sec 2024-11-24T20:48:19,858 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:19,858 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72252cc9a6c51612da375fd8db66c697:B 2024-11-24T20:48:19,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:19,898 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112400b712bd490f4bc3b24b7eb4e21311f5_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112400b712bd490f4bc3b24b7eb4e21311f5_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:19,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/936943eed7fe4c67b7ac8a6ffcf09b70, store: [table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:19,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/936943eed7fe4c67b7ac8a6ffcf09b70 is 175, key is test_row_0/A:col10/1732481297981/Put/seqid=0 2024-11-24T20:48:19,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742175_1351 (size=31105) 2024-11-24T20:48:19,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T20:48:20,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:20,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:20,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:20,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481360203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:20,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:20,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:20,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:20,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:20,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481360204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:20,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:20,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:20,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:20,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481360205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:20,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:20,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:20,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481360205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:20,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:20,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:20,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:20,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:20,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:20,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:20,309 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/936943eed7fe4c67b7ac8a6ffcf09b70 2024-11-24T20:48:20,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/c0605d6ac5e44f37b8e56f4f30c967b3 is 50, key is test_row_0/B:col10/1732481297981/Put/seqid=0 2024-11-24T20:48:20,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:20,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481360315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:20,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481360315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:20,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:20,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:20,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:20,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:20,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:20,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:20,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742176_1352 (size=12151) 2024-11-24T20:48:20,324 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/c0605d6ac5e44f37b8e56f4f30c967b3 2024-11-24T20:48:20,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:20,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481360326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:20,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:20,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:20,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481360326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:20,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:20,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:20,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:20,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:20,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:20,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/299ff3d8b4584ff7ac574508362542e4 is 50, key is test_row_0/C:col10/1732481297981/Put/seqid=0 2024-11-24T20:48:20,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742177_1353 (size=12151) 2024-11-24T20:48:20,444 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T20:48:20,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:20,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481360525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:20,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:20,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481360525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:20,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:20,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:20,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:20,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:20,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:20,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:20,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:20,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481360536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:20,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:20,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:20,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:20,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:20,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481360545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:20,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:20,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:20,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:20,743 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/299ff3d8b4584ff7ac574508362542e4 2024-11-24T20:48:20,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/936943eed7fe4c67b7ac8a6ffcf09b70 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/936943eed7fe4c67b7ac8a6ffcf09b70 2024-11-24T20:48:20,749 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/936943eed7fe4c67b7ac8a6ffcf09b70, entries=150, sequenceid=168, filesize=30.4 K 2024-11-24T20:48:20,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/c0605d6ac5e44f37b8e56f4f30c967b3 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/c0605d6ac5e44f37b8e56f4f30c967b3 2024-11-24T20:48:20,754 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/c0605d6ac5e44f37b8e56f4f30c967b3, entries=150, sequenceid=168, filesize=11.9 K 2024-11-24T20:48:20,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/299ff3d8b4584ff7ac574508362542e4 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/299ff3d8b4584ff7ac574508362542e4 2024-11-24T20:48:20,758 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/299ff3d8b4584ff7ac574508362542e4, entries=150, sequenceid=168, filesize=11.9 K 2024-11-24T20:48:20,759 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 72252cc9a6c51612da375fd8db66c697 in 1404ms, sequenceid=168, compaction requested=false 2024-11-24T20:48:20,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:20,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:20,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-24T20:48:20,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-24T20:48:20,762 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-24T20:48:20,762 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9400 sec 2024-11-24T20:48:20,765 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.9470 sec 2024-11-24T20:48:20,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:20,839 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:48:20,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72252cc9a6c51612da375fd8db66c697 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-24T20:48:20,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411248ff4ea689187417a932e5a2cf980d43d_72252cc9a6c51612da375fd8db66c697 is 50, key is test_row_0/A:col10/1732481300838/Put/seqid=0 2024-11-24T20:48:20,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742178_1354 (size=14794) 2024-11-24T20:48:20,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481360893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:20,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481360893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:20,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:20,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481360903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:20,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:20,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:20,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:20,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481360905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:20,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:20,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:20,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:21,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:21,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481361015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:21,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:21,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:21,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:21,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481361015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:21,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:21,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:21,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481361016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:21,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:21,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:21,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:21,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:21,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:21,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:21,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:21,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481361025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:21,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:21,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:21,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:21,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:21,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481361225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:21,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:21,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481361226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:21,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:21,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:21,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481361226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:21,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:21,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481361235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:21,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:21,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:21,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:21,251 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:21,255 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411248ff4ea689187417a932e5a2cf980d43d_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248ff4ea689187417a932e5a2cf980d43d_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:21,256 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/75694e5737f54fa4b2361639c279d0a1, store: [table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:21,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/75694e5737f54fa4b2361639c279d0a1 is 175, key is test_row_0/A:col10/1732481300838/Put/seqid=0 2024-11-24T20:48:21,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742179_1355 (size=39749) 2024-11-24T20:48:21,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:21,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481361534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:21,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:21,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:21,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:21,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:21,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481361535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:21,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:21,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:21,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:21,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:21,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481361535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:21,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:21,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:21,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:21,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:21,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481361545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:21,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:21,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:21,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:21,665 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=183, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/75694e5737f54fa4b2361639c279d0a1 2024-11-24T20:48:21,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/048ca060dbcc4ab0a547dbf96a6a5b1e is 50, key is test_row_0/B:col10/1732481300838/Put/seqid=0 2024-11-24T20:48:21,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742180_1356 (size=12151) 2024-11-24T20:48:21,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/048ca060dbcc4ab0a547dbf96a6a5b1e 2024-11-24T20:48:21,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/bede6f8f520d4a009b806be5d49bb023 is 50, key is test_row_0/C:col10/1732481300838/Put/seqid=0 2024-11-24T20:48:21,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742181_1357 (size=12151) 2024-11-24T20:48:21,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T20:48:21,985 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:48:21,986 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:48:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-24T20:48:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T20:48:21,987 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:48:21,988 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:48:21,988 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:48:22,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481362055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:22,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481362055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:22,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481362055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:22,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:22,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481362065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:22,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/bede6f8f520d4a009b806be5d49bb023 2024-11-24T20:48:22,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T20:48:22,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/75694e5737f54fa4b2361639c279d0a1 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/75694e5737f54fa4b2361639c279d0a1 2024-11-24T20:48:22,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/75694e5737f54fa4b2361639c279d0a1, entries=200, sequenceid=183, filesize=38.8 K 2024-11-24T20:48:22,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/048ca060dbcc4ab0a547dbf96a6a5b1e as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/048ca060dbcc4ab0a547dbf96a6a5b1e 2024-11-24T20:48:22,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/048ca060dbcc4ab0a547dbf96a6a5b1e, entries=150, sequenceid=183, filesize=11.9 K 2024-11-24T20:48:22,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/bede6f8f520d4a009b806be5d49bb023 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/bede6f8f520d4a009b806be5d49bb023 2024-11-24T20:48:22,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/bede6f8f520d4a009b806be5d49bb023, entries=150, sequenceid=183, filesize=11.9 K 2024-11-24T20:48:22,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 72252cc9a6c51612da375fd8db66c697 in 1409ms, sequenceid=183, compaction requested=true 2024-11-24T20:48:22,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:22,248 INFO [AsyncFSWAL-0-hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a-prefix:20b981ef5c27,34625,1732481182291 {}] wal.AbstractFSWAL(1368): Slow sync cost: 138 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40053,DS-6a47c115-3646-4421-9ff7-0ca9cf144cbe,DISK]] 2024-11-24T20:48:22,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72252cc9a6c51612da375fd8db66c697:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:48:22,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:22,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72252cc9a6c51612da375fd8db66c697:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:48:22,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-24T20:48:22,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:22,249 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:22,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72252cc9a6c51612da375fd8db66c697:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:48:22,249 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:22,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:22,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:22,249 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 72252cc9a6c51612da375fd8db66c697 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-24T20:48:22,251 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:22,251 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:22,251 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 72252cc9a6c51612da375fd8db66c697/B is initiating minor compaction (all files) 2024-11-24T20:48:22,251 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 72252cc9a6c51612da375fd8db66c697/A is initiating minor compaction (all files) 2024-11-24T20:48:22,252 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72252cc9a6c51612da375fd8db66c697/B in TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:22,252 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72252cc9a6c51612da375fd8db66c697/A in TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:22,252 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/3c3e248dd0c94e83ad630e94eec28aa8, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/c0605d6ac5e44f37b8e56f4f30c967b3, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/048ca060dbcc4ab0a547dbf96a6a5b1e] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp, totalSize=35.9 K 2024-11-24T20:48:22,252 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/479fd04a93054a4ba74e2d8ec98ea5ab, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/936943eed7fe4c67b7ac8a6ffcf09b70, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/75694e5737f54fa4b2361639c279d0a1] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp, totalSize=99.9 K 2024-11-24T20:48:22,252 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:22,252 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. files: [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/479fd04a93054a4ba74e2d8ec98ea5ab, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/936943eed7fe4c67b7ac8a6ffcf09b70, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/75694e5737f54fa4b2361639c279d0a1] 2024-11-24T20:48:22,253 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 479fd04a93054a4ba74e2d8ec98ea5ab, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732481297897 2024-11-24T20:48:22,253 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c3e248dd0c94e83ad630e94eec28aa8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732481297897 2024-11-24T20:48:22,254 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 936943eed7fe4c67b7ac8a6ffcf09b70, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732481297980 2024-11-24T20:48:22,254 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting c0605d6ac5e44f37b8e56f4f30c967b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732481297980 2024-11-24T20:48:22,255 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75694e5737f54fa4b2361639c279d0a1, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732481300183 2024-11-24T20:48:22,255 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 048ca060dbcc4ab0a547dbf96a6a5b1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732481300183 2024-11-24T20:48:22,265 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:22,266 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72252cc9a6c51612da375fd8db66c697#B#compaction#303 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:22,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124debb5068d77d493da1c172c0d145db64_72252cc9a6c51612da375fd8db66c697 is 50, key is test_row_0/A:col10/1732481300901/Put/seqid=0 2024-11-24T20:48:22,267 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/be5d00a4c58e423cb2d394362698f8fa is 50, key is test_row_0/B:col10/1732481300838/Put/seqid=0 2024-11-24T20:48:22,270 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411246c28a983897d4c6e91a916719ce1546d_72252cc9a6c51612da375fd8db66c697 store=[table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:22,271 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411246c28a983897d4c6e91a916719ce1546d_72252cc9a6c51612da375fd8db66c697, store=[table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:22,271 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411246c28a983897d4c6e91a916719ce1546d_72252cc9a6c51612da375fd8db66c697 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:22,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742182_1358 (size=12304) 2024-11-24T20:48:22,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742183_1359 (size=12595) 2024-11-24T20:48:22,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742184_1360 (size=4469) 2024-11-24T20:48:22,285 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72252cc9a6c51612da375fd8db66c697#A#compaction#304 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:22,285 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/25e8508ef41f4889b6ed66f12f6f80a7 is 175, key is test_row_0/A:col10/1732481300838/Put/seqid=0 2024-11-24T20:48:22,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742185_1361 (size=31549) 2024-11-24T20:48:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T20:48:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T20:48:22,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:22,684 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/be5d00a4c58e423cb2d394362698f8fa as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/be5d00a4c58e423cb2d394362698f8fa 2024-11-24T20:48:22,684 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124debb5068d77d493da1c172c0d145db64_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124debb5068d77d493da1c172c0d145db64_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:22,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/ffd9ef9cc3aa41faad351b4ab34823ba, store: [table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:22,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/ffd9ef9cc3aa41faad351b4ab34823ba is 175, key is test_row_0/A:col10/1732481300901/Put/seqid=0 2024-11-24T20:48:22,691 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72252cc9a6c51612da375fd8db66c697/B of 72252cc9a6c51612da375fd8db66c697 into be5d00a4c58e423cb2d394362698f8fa(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:22,691 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:22,691 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., storeName=72252cc9a6c51612da375fd8db66c697/B, priority=13, startTime=1732481302249; duration=0sec 2024-11-24T20:48:22,692 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:22,692 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72252cc9a6c51612da375fd8db66c697:B 2024-11-24T20:48:22,692 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:22,693 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:22,694 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 72252cc9a6c51612da375fd8db66c697/C is initiating minor compaction (all files) 2024-11-24T20:48:22,694 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72252cc9a6c51612da375fd8db66c697/C in TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:22,694 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/080ac9cf3a764fa6a904be634bb7f81f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/299ff3d8b4584ff7ac574508362542e4, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/bede6f8f520d4a009b806be5d49bb023] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp, totalSize=35.9 K 2024-11-24T20:48:22,694 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 080ac9cf3a764fa6a904be634bb7f81f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732481297897 2024-11-24T20:48:22,695 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 299ff3d8b4584ff7ac574508362542e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732481297980 2024-11-24T20:48:22,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742186_1362 (size=31105) 2024-11-24T20:48:22,696 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting bede6f8f520d4a009b806be5d49bb023, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732481300183 2024-11-24T20:48:22,697 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/25e8508ef41f4889b6ed66f12f6f80a7 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/25e8508ef41f4889b6ed66f12f6f80a7 2024-11-24T20:48:22,697 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/ffd9ef9cc3aa41faad351b4ab34823ba 2024-11-24T20:48:22,707 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72252cc9a6c51612da375fd8db66c697/A of 72252cc9a6c51612da375fd8db66c697 into 25e8508ef41f4889b6ed66f12f6f80a7(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:22,707 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:22,707 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., storeName=72252cc9a6c51612da375fd8db66c697/A, priority=13, startTime=1732481302248; duration=0sec 2024-11-24T20:48:22,707 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:22,707 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72252cc9a6c51612da375fd8db66c697:A 2024-11-24T20:48:22,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/56eea5de41f84db9ba765e4a7a859297 is 50, key is test_row_0/B:col10/1732481300901/Put/seqid=0 2024-11-24T20:48:22,714 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72252cc9a6c51612da375fd8db66c697#C#compaction#306 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:22,715 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/7584f6f2eb05460f8438be36f4ccfaed is 50, key is test_row_0/C:col10/1732481300838/Put/seqid=0 2024-11-24T20:48:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:22,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:22,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742187_1363 (size=12151) 2024-11-24T20:48:22,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742188_1364 (size=12595) 2024-11-24T20:48:22,734 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/7584f6f2eb05460f8438be36f4ccfaed as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/7584f6f2eb05460f8438be36f4ccfaed 2024-11-24T20:48:22,740 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72252cc9a6c51612da375fd8db66c697/C of 72252cc9a6c51612da375fd8db66c697 into 7584f6f2eb05460f8438be36f4ccfaed(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:22,740 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:22,740 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., storeName=72252cc9a6c51612da375fd8db66c697/C, priority=13, startTime=1732481302249; duration=0sec 2024-11-24T20:48:22,740 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:22,740 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72252cc9a6c51612da375fd8db66c697:C 2024-11-24T20:48:22,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481362786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:22,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481362895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:22,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:22,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:22,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:23,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481363076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:23,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481363076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:23,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481363076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:23,079 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_1' in 72252cc9a6c51612da375fd8db66c697 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:48:23,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:23,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:23,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:23,079 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_0' in 72252cc9a6c51612da375fd8db66c697 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:48:23,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:23,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:23,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:23,080 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_2' in 72252cc9a6c51612da375fd8db66c697 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:48:23,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:23,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:23,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:23,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481363086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:23,090 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_2' in 72252cc9a6c51612da375fd8db66c697 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:48:23,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:23,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:23,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:23,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481363115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:23,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:23,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:23,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:23,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T20:48:23,126 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/56eea5de41f84db9ba765e4a7a859297 2024-11-24T20:48:23,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/955f3e0ede10473ab1dc833c2bc68c3d is 50, key is test_row_0/C:col10/1732481300901/Put/seqid=0 2024-11-24T20:48:23,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742189_1365 (size=12151) 2024-11-24T20:48:23,159 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/955f3e0ede10473ab1dc833c2bc68c3d 2024-11-24T20:48:23,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/ffd9ef9cc3aa41faad351b4ab34823ba as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/ffd9ef9cc3aa41faad351b4ab34823ba 2024-11-24T20:48:23,181 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/ffd9ef9cc3aa41faad351b4ab34823ba, entries=150, sequenceid=207, filesize=30.4 K 2024-11-24T20:48:23,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/56eea5de41f84db9ba765e4a7a859297 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/56eea5de41f84db9ba765e4a7a859297 2024-11-24T20:48:23,186 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/56eea5de41f84db9ba765e4a7a859297, entries=150, sequenceid=207, filesize=11.9 K 2024-11-24T20:48:23,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/955f3e0ede10473ab1dc833c2bc68c3d as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/955f3e0ede10473ab1dc833c2bc68c3d 2024-11-24T20:48:23,196 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/955f3e0ede10473ab1dc833c2bc68c3d, entries=150, sequenceid=207, filesize=11.9 K 2024-11-24T20:48:23,197 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 72252cc9a6c51612da375fd8db66c697 in 948ms, sequenceid=207, compaction requested=false 2024-11-24T20:48:23,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:23,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:23,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-24T20:48:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-24T20:48:23,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-24T20:48:23,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2130 sec 2024-11-24T20:48:23,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.2180 sec 2024-11-24T20:48:23,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:23,432 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:48:23,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72252cc9a6c51612da375fd8db66c697 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-24T20:48:23,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112489130d66d7904644b6ab4700a25cafc7_72252cc9a6c51612da375fd8db66c697 is 50, key is test_row_0/A:col10/1732481302784/Put/seqid=0 2024-11-24T20:48:23,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742190_1366 (size=14794) 2024-11-24T20:48:23,448 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:23,452 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112489130d66d7904644b6ab4700a25cafc7_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112489130d66d7904644b6ab4700a25cafc7_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:23,453 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/b9629e598aaa404db2a88ac488db27fd, store: [table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:23,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/b9629e598aaa404db2a88ac488db27fd is 175, key is test_row_0/A:col10/1732481302784/Put/seqid=0 2024-11-24T20:48:23,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742191_1367 (size=39749) 2024-11-24T20:48:23,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:23,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481363612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:23,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:23,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:23,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:23,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:23,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481363725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:23,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:23,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:23,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:23,866 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=223, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/b9629e598aaa404db2a88ac488db27fd 2024-11-24T20:48:23,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/bfe36eb2e522479a8752b7b573e13baa is 50, key is test_row_0/B:col10/1732481302784/Put/seqid=0 2024-11-24T20:48:23,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742192_1368 (size=12151) 2024-11-24T20:48:23,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/bfe36eb2e522479a8752b7b573e13baa 2024-11-24T20:48:23,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/141ca43321ec4bfdbc4fa04b9db51cf9 is 50, key is test_row_0/C:col10/1732481302784/Put/seqid=0 2024-11-24T20:48:23,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742193_1369 (size=12151) 2024-11-24T20:48:23,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/141ca43321ec4bfdbc4fa04b9db51cf9 2024-11-24T20:48:23,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/b9629e598aaa404db2a88ac488db27fd as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/b9629e598aaa404db2a88ac488db27fd 2024-11-24T20:48:23,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/b9629e598aaa404db2a88ac488db27fd, entries=200, sequenceid=223, filesize=38.8 K 2024-11-24T20:48:23,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/bfe36eb2e522479a8752b7b573e13baa as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/bfe36eb2e522479a8752b7b573e13baa 2024-11-24T20:48:23,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/bfe36eb2e522479a8752b7b573e13baa, entries=150, sequenceid=223, filesize=11.9 K 2024-11-24T20:48:23,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/141ca43321ec4bfdbc4fa04b9db51cf9 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/141ca43321ec4bfdbc4fa04b9db51cf9 2024-11-24T20:48:23,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/141ca43321ec4bfdbc4fa04b9db51cf9, entries=150, sequenceid=223, filesize=11.9 K 2024-11-24T20:48:23,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 72252cc9a6c51612da375fd8db66c697 in 498ms, sequenceid=223, compaction requested=true 2024-11-24T20:48:23,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:23,930 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:23,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72252cc9a6c51612da375fd8db66c697:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:48:23,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:23,931 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:23,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72252cc9a6c51612da375fd8db66c697:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:48:23,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:23,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72252cc9a6c51612da375fd8db66c697:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:48:23,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:23,931 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:23,931 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 72252cc9a6c51612da375fd8db66c697/A is initiating minor compaction (all files) 2024-11-24T20:48:23,931 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72252cc9a6c51612da375fd8db66c697/A in TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:23,931 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/25e8508ef41f4889b6ed66f12f6f80a7, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/ffd9ef9cc3aa41faad351b4ab34823ba, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/b9629e598aaa404db2a88ac488db27fd] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp, totalSize=100.0 K 2024-11-24T20:48:23,931 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:23,931 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. files: [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/25e8508ef41f4889b6ed66f12f6f80a7, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/ffd9ef9cc3aa41faad351b4ab34823ba, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/b9629e598aaa404db2a88ac488db27fd] 2024-11-24T20:48:23,939 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25e8508ef41f4889b6ed66f12f6f80a7, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732481300183 2024-11-24T20:48:23,939 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffd9ef9cc3aa41faad351b4ab34823ba, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732481300892 2024-11-24T20:48:23,940 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:23,940 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 72252cc9a6c51612da375fd8db66c697/B is initiating minor compaction (all files) 2024-11-24T20:48:23,940 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72252cc9a6c51612da375fd8db66c697/B in TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:23,940 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/be5d00a4c58e423cb2d394362698f8fa, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/56eea5de41f84db9ba765e4a7a859297, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/bfe36eb2e522479a8752b7b573e13baa] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp, totalSize=36.0 K 2024-11-24T20:48:23,940 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting be5d00a4c58e423cb2d394362698f8fa, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732481300183 2024-11-24T20:48:23,940 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9629e598aaa404db2a88ac488db27fd, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732481302768 2024-11-24T20:48:23,940 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 56eea5de41f84db9ba765e4a7a859297, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732481300892 2024-11-24T20:48:23,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:23,941 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting bfe36eb2e522479a8752b7b573e13baa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732481302776 2024-11-24T20:48:23,941 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:48:23,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72252cc9a6c51612da375fd8db66c697 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-24T20:48:23,949 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72252cc9a6c51612da375fd8db66c697#B#compaction#311 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:23,950 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/dacb89c276fb4f47a5b4c9820ba0143c is 50, key is test_row_0/B:col10/1732481302784/Put/seqid=0 2024-11-24T20:48:23,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244658ead7a82742ecb87b6a6b1bd5973e_72252cc9a6c51612da375fd8db66c697 is 50, key is test_row_0/A:col10/1732481303588/Put/seqid=0 2024-11-24T20:48:23,957 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:23,973 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124778b8290e7974d6fb6e8527127975cfc_72252cc9a6c51612da375fd8db66c697 store=[table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:23,975 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124778b8290e7974d6fb6e8527127975cfc_72252cc9a6c51612da375fd8db66c697, store=[table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:23,975 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124778b8290e7974d6fb6e8527127975cfc_72252cc9a6c51612da375fd8db66c697 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:23,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742194_1370 (size=12697) 2024-11-24T20:48:23,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742195_1371 (size=14794) 2024-11-24T20:48:23,984 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/dacb89c276fb4f47a5b4c9820ba0143c as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/dacb89c276fb4f47a5b4c9820ba0143c 2024-11-24T20:48:23,984 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:23,987 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244658ead7a82742ecb87b6a6b1bd5973e_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244658ead7a82742ecb87b6a6b1bd5973e_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:23,988 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/c16f4d08b8ea4ed3a4719c6abdeb3c0b, store: [table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:23,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/c16f4d08b8ea4ed3a4719c6abdeb3c0b is 175, key is test_row_0/A:col10/1732481303588/Put/seqid=0 2024-11-24T20:48:23,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742196_1372 (size=4469) 2024-11-24T20:48:23,992 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72252cc9a6c51612da375fd8db66c697#A#compaction#313 average throughput is 0.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:23,992 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/3d7f27e014d14edda6ea103a7d3d761d is 175, key is test_row_0/A:col10/1732481302784/Put/seqid=0 2024-11-24T20:48:23,992 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72252cc9a6c51612da375fd8db66c697/B of 72252cc9a6c51612da375fd8db66c697 into dacb89c276fb4f47a5b4c9820ba0143c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:23,993 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:23,993 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., storeName=72252cc9a6c51612da375fd8db66c697/B, priority=13, startTime=1732481303931; duration=0sec 2024-11-24T20:48:23,993 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:23,993 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72252cc9a6c51612da375fd8db66c697:B 2024-11-24T20:48:23,993 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:23,994 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:23,994 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 72252cc9a6c51612da375fd8db66c697/C is initiating minor compaction (all files) 2024-11-24T20:48:23,994 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72252cc9a6c51612da375fd8db66c697/C in TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:23,994 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/7584f6f2eb05460f8438be36f4ccfaed, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/955f3e0ede10473ab1dc833c2bc68c3d, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/141ca43321ec4bfdbc4fa04b9db51cf9] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp, totalSize=36.0 K 2024-11-24T20:48:23,997 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 7584f6f2eb05460f8438be36f4ccfaed, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732481300183 2024-11-24T20:48:23,998 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 955f3e0ede10473ab1dc833c2bc68c3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732481300892 2024-11-24T20:48:23,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742198_1374 (size=31651) 2024-11-24T20:48:23,999 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 141ca43321ec4bfdbc4fa04b9db51cf9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732481302776 2024-11-24T20:48:24,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742197_1373 (size=39749) 2024-11-24T20:48:24,002 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/c16f4d08b8ea4ed3a4719c6abdeb3c0b 2024-11-24T20:48:24,007 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72252cc9a6c51612da375fd8db66c697#C#compaction#314 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:24,007 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/fb004d88026c456597efd2004b35fb84 is 50, key is test_row_0/C:col10/1732481302784/Put/seqid=0 2024-11-24T20:48:24,007 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/3d7f27e014d14edda6ea103a7d3d761d as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/3d7f27e014d14edda6ea103a7d3d761d 2024-11-24T20:48:24,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742199_1375 (size=12697) 2024-11-24T20:48:24,015 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72252cc9a6c51612da375fd8db66c697/A of 72252cc9a6c51612da375fd8db66c697 into 3d7f27e014d14edda6ea103a7d3d761d(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:24,016 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:24,016 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., storeName=72252cc9a6c51612da375fd8db66c697/A, priority=13, startTime=1732481303930; duration=0sec 2024-11-24T20:48:24,016 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:24,016 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72252cc9a6c51612da375fd8db66c697:A 2024-11-24T20:48:24,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/628397602fbb46039e09a05b1c118110 is 50, key is test_row_0/B:col10/1732481303588/Put/seqid=0 2024-11-24T20:48:24,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481364009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:24,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742200_1376 (size=12151) 2024-11-24T20:48:24,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:24,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481364125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:24,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:24,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:24,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T20:48:24,135 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:48:24,135 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:48:24,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-24T20:48:24,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T20:48:24,137 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:48:24,137 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:48:24,137 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:48:24,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T20:48:24,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-24T20:48:24,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:24,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:24,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:24,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:24,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:24,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:24,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:24,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481364335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:24,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:24,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:24,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:24,418 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/fb004d88026c456597efd2004b35fb84 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/fb004d88026c456597efd2004b35fb84 2024-11-24T20:48:24,422 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72252cc9a6c51612da375fd8db66c697/C of 72252cc9a6c51612da375fd8db66c697 into fb004d88026c456597efd2004b35fb84(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:24,422 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:24,422 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., storeName=72252cc9a6c51612da375fd8db66c697/C, priority=13, startTime=1732481303931; duration=0sec 2024-11-24T20:48:24,422 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:24,422 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72252cc9a6c51612da375fd8db66c697:C 2024-11-24T20:48:24,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/628397602fbb46039e09a05b1c118110 2024-11-24T20:48:24,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/cbbc46cfac2c404da93057d9530a9cd1 is 50, key is test_row_0/C:col10/1732481303588/Put/seqid=0 2024-11-24T20:48:24,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742201_1377 (size=12151) 2024-11-24T20:48:24,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-24T20:48:24,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:24,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:24,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:24,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:24,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:24,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:24,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T20:48:24,499 WARN [regionserver/20b981ef5c27:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 0 2024-11-24T20:48:24,502 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/218af5c4e3724f74b3b35a9707539a08, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/348e5f4f822842c4bebf805c2963952f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/462360228d564326a4ec4f5238e826f8, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/0c7a178dc21d47c39ef39b2f5160b801, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/1317fe1c369145c38222d3cf44b258de, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/e045dd7cc07f4bb39f543be5d9c56ee4, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/068f10e08df941d191983c7e04648326, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/6f675e6880314028aeba0f18b346d3fc, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/8d4ac2e8bec14543a8079c7390ce1290, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/df2b580779014fb1b67e16074a8fa46a, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/479fd04a93054a4ba74e2d8ec98ea5ab, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/936943eed7fe4c67b7ac8a6ffcf09b70, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/75694e5737f54fa4b2361639c279d0a1, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/25e8508ef41f4889b6ed66f12f6f80a7, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/ffd9ef9cc3aa41faad351b4ab34823ba, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/b9629e598aaa404db2a88ac488db27fd] to archive 2024-11-24T20:48:24,502 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T20:48:24,504 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/218af5c4e3724f74b3b35a9707539a08 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/218af5c4e3724f74b3b35a9707539a08 2024-11-24T20:48:24,505 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/348e5f4f822842c4bebf805c2963952f to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/348e5f4f822842c4bebf805c2963952f 2024-11-24T20:48:24,506 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/462360228d564326a4ec4f5238e826f8 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/462360228d564326a4ec4f5238e826f8 2024-11-24T20:48:24,508 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/0c7a178dc21d47c39ef39b2f5160b801 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/0c7a178dc21d47c39ef39b2f5160b801 2024-11-24T20:48:24,509 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/1317fe1c369145c38222d3cf44b258de to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/1317fe1c369145c38222d3cf44b258de 2024-11-24T20:48:24,509 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/e045dd7cc07f4bb39f543be5d9c56ee4 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/e045dd7cc07f4bb39f543be5d9c56ee4 2024-11-24T20:48:24,510 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/068f10e08df941d191983c7e04648326 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/068f10e08df941d191983c7e04648326 2024-11-24T20:48:24,511 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/6f675e6880314028aeba0f18b346d3fc to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/6f675e6880314028aeba0f18b346d3fc 2024-11-24T20:48:24,512 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/8d4ac2e8bec14543a8079c7390ce1290 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/8d4ac2e8bec14543a8079c7390ce1290 2024-11-24T20:48:24,513 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/df2b580779014fb1b67e16074a8fa46a to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/df2b580779014fb1b67e16074a8fa46a 2024-11-24T20:48:24,514 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/479fd04a93054a4ba74e2d8ec98ea5ab to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/479fd04a93054a4ba74e2d8ec98ea5ab 2024-11-24T20:48:24,516 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/936943eed7fe4c67b7ac8a6ffcf09b70 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/936943eed7fe4c67b7ac8a6ffcf09b70 2024-11-24T20:48:24,517 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/75694e5737f54fa4b2361639c279d0a1 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/75694e5737f54fa4b2361639c279d0a1 2024-11-24T20:48:24,518 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/25e8508ef41f4889b6ed66f12f6f80a7 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/25e8508ef41f4889b6ed66f12f6f80a7 2024-11-24T20:48:24,519 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/ffd9ef9cc3aa41faad351b4ab34823ba to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/ffd9ef9cc3aa41faad351b4ab34823ba 2024-11-24T20:48:24,520 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/b9629e598aaa404db2a88ac488db27fd to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/b9629e598aaa404db2a88ac488db27fd 2024-11-24T20:48:24,523 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/1bb5c213775545359a73cef659115066, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/14d1396f99254d869c66d87ae0a56718, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/3eb3bfbb35e34238801178f39f5526ab, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/295b194b00eb41fc89340ec209038fa6, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/0e3321453d1e43449def4800a95d2176, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/9d0d7aed725c43be975986ca948b95ca, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/6e7bb6b8e7344f0088d5764c3218b7ea, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/60f581b50eb343cdbfb8c5d8cb949414, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/cd22368b33c84fc6b39ae92347db7e3b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/3c3e248dd0c94e83ad630e94eec28aa8, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/fb2fc177dda54cbca972de7ce4f74488, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/c0605d6ac5e44f37b8e56f4f30c967b3, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/be5d00a4c58e423cb2d394362698f8fa, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/048ca060dbcc4ab0a547dbf96a6a5b1e, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/56eea5de41f84db9ba765e4a7a859297, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/bfe36eb2e522479a8752b7b573e13baa] to archive 2024-11-24T20:48:24,524 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T20:48:24,525 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/1bb5c213775545359a73cef659115066 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/1bb5c213775545359a73cef659115066 2024-11-24T20:48:24,526 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/14d1396f99254d869c66d87ae0a56718 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/14d1396f99254d869c66d87ae0a56718 2024-11-24T20:48:24,527 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/3eb3bfbb35e34238801178f39f5526ab to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/3eb3bfbb35e34238801178f39f5526ab 2024-11-24T20:48:24,528 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/295b194b00eb41fc89340ec209038fa6 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/295b194b00eb41fc89340ec209038fa6 2024-11-24T20:48:24,529 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/0e3321453d1e43449def4800a95d2176 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/0e3321453d1e43449def4800a95d2176 2024-11-24T20:48:24,529 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/9d0d7aed725c43be975986ca948b95ca to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/9d0d7aed725c43be975986ca948b95ca 2024-11-24T20:48:24,530 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/6e7bb6b8e7344f0088d5764c3218b7ea to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/6e7bb6b8e7344f0088d5764c3218b7ea 2024-11-24T20:48:24,531 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/60f581b50eb343cdbfb8c5d8cb949414 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/60f581b50eb343cdbfb8c5d8cb949414 2024-11-24T20:48:24,532 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/cd22368b33c84fc6b39ae92347db7e3b to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/cd22368b33c84fc6b39ae92347db7e3b 2024-11-24T20:48:24,533 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/3c3e248dd0c94e83ad630e94eec28aa8 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/3c3e248dd0c94e83ad630e94eec28aa8 2024-11-24T20:48:24,533 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/fb2fc177dda54cbca972de7ce4f74488 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/fb2fc177dda54cbca972de7ce4f74488 2024-11-24T20:48:24,534 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/c0605d6ac5e44f37b8e56f4f30c967b3 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/c0605d6ac5e44f37b8e56f4f30c967b3 2024-11-24T20:48:24,535 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/be5d00a4c58e423cb2d394362698f8fa to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/be5d00a4c58e423cb2d394362698f8fa 2024-11-24T20:48:24,535 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/048ca060dbcc4ab0a547dbf96a6a5b1e to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/048ca060dbcc4ab0a547dbf96a6a5b1e 2024-11-24T20:48:24,536 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/56eea5de41f84db9ba765e4a7a859297 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/56eea5de41f84db9ba765e4a7a859297 2024-11-24T20:48:24,537 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/bfe36eb2e522479a8752b7b573e13baa to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/bfe36eb2e522479a8752b7b573e13baa 2024-11-24T20:48:24,540 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/06ea0b9cb3a246b383c292420b5e9260, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/7d45786e1b7a4b9ea64fbe660a6a6cd9, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/0f76ec902c704659bcb167d8a36801f8, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/27732c7e3d4346e29935d08c3d6d1ff7, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/eb3d6c49a98e4f2bb2a5a02e9ce79318, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/a85aa1a9680c4ad78bbcfd335f1a69d1, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/9739328e78ef4af09f13384b352ed71a, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/6965fdc7e71a497c8efc5f46f7196113, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/968672f540e24e50a799506ed3e5791b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/080ac9cf3a764fa6a904be634bb7f81f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/5fbb96e3083e46e888da91ae39a484ac, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/299ff3d8b4584ff7ac574508362542e4, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/7584f6f2eb05460f8438be36f4ccfaed, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/bede6f8f520d4a009b806be5d49bb023, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/955f3e0ede10473ab1dc833c2bc68c3d, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/141ca43321ec4bfdbc4fa04b9db51cf9] to archive 2024-11-24T20:48:24,540 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T20:48:24,542 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/06ea0b9cb3a246b383c292420b5e9260 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/06ea0b9cb3a246b383c292420b5e9260 2024-11-24T20:48:24,542 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/7d45786e1b7a4b9ea64fbe660a6a6cd9 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/7d45786e1b7a4b9ea64fbe660a6a6cd9 2024-11-24T20:48:24,543 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/0f76ec902c704659bcb167d8a36801f8 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/0f76ec902c704659bcb167d8a36801f8 2024-11-24T20:48:24,544 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/27732c7e3d4346e29935d08c3d6d1ff7 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/27732c7e3d4346e29935d08c3d6d1ff7 2024-11-24T20:48:24,545 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/eb3d6c49a98e4f2bb2a5a02e9ce79318 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/eb3d6c49a98e4f2bb2a5a02e9ce79318 2024-11-24T20:48:24,546 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/a85aa1a9680c4ad78bbcfd335f1a69d1 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/a85aa1a9680c4ad78bbcfd335f1a69d1 2024-11-24T20:48:24,546 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/9739328e78ef4af09f13384b352ed71a to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/9739328e78ef4af09f13384b352ed71a 2024-11-24T20:48:24,547 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/6965fdc7e71a497c8efc5f46f7196113 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/6965fdc7e71a497c8efc5f46f7196113 2024-11-24T20:48:24,548 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/968672f540e24e50a799506ed3e5791b to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/968672f540e24e50a799506ed3e5791b 2024-11-24T20:48:24,549 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/080ac9cf3a764fa6a904be634bb7f81f to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/080ac9cf3a764fa6a904be634bb7f81f 2024-11-24T20:48:24,549 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/5fbb96e3083e46e888da91ae39a484ac to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/5fbb96e3083e46e888da91ae39a484ac 2024-11-24T20:48:24,550 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/299ff3d8b4584ff7ac574508362542e4 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/299ff3d8b4584ff7ac574508362542e4 2024-11-24T20:48:24,551 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/7584f6f2eb05460f8438be36f4ccfaed to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/7584f6f2eb05460f8438be36f4ccfaed 2024-11-24T20:48:24,553 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/bede6f8f520d4a009b806be5d49bb023 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/bede6f8f520d4a009b806be5d49bb023 2024-11-24T20:48:24,554 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/955f3e0ede10473ab1dc833c2bc68c3d to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/955f3e0ede10473ab1dc833c2bc68c3d 2024-11-24T20:48:24,555 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/141ca43321ec4bfdbc4fa04b9db51cf9 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/141ca43321ec4bfdbc4fa04b9db51cf9 2024-11-24T20:48:24,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-24T20:48:24,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:24,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:24,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:24,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:24,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:24,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:24,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481364645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:24,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:24,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:24,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:24,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-24T20:48:24,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:24,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:24,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:24,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:24,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T20:48:24,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/cbbc46cfac2c404da93057d9530a9cd1 2024-11-24T20:48:24,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/c16f4d08b8ea4ed3a4719c6abdeb3c0b as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/c16f4d08b8ea4ed3a4719c6abdeb3c0b 2024-11-24T20:48:24,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/c16f4d08b8ea4ed3a4719c6abdeb3c0b, entries=200, sequenceid=247, filesize=38.8 K 2024-11-24T20:48:24,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/628397602fbb46039e09a05b1c118110 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/628397602fbb46039e09a05b1c118110 2024-11-24T20:48:24,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/628397602fbb46039e09a05b1c118110, entries=150, sequenceid=247, filesize=11.9 K 2024-11-24T20:48:24,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/cbbc46cfac2c404da93057d9530a9cd1 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/cbbc46cfac2c404da93057d9530a9cd1 2024-11-24T20:48:24,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/cbbc46cfac2c404da93057d9530a9cd1, entries=150, sequenceid=247, filesize=11.9 K 2024-11-24T20:48:24,855 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 72252cc9a6c51612da375fd8db66c697 in 914ms, sequenceid=247, compaction requested=false 2024-11-24T20:48:24,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:24,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-24T20:48:24,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:24,902 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 72252cc9a6c51612da375fd8db66c697 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-24T20:48:24,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124399704b707f446abb13c112e5d891018_72252cc9a6c51612da375fd8db66c697 is 50, key is test_row_0/A:col10/1732481303983/Put/seqid=0 2024-11-24T20:48:24,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742202_1378 (size=12404) 2024-11-24T20:48:25,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:25,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:25,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:25,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481365151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:25,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:25,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481365152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:25,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:25,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:25,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:25,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481365154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:25,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:25,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:25,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481365156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:25,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:25,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:25,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481365165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:25,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T20:48:25,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:25,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481365276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:25,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:25,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481365276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:25,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:25,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481365276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:25,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:25,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:25,281 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:25,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481365276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:25,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:25,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:25,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:25,351 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124399704b707f446abb13c112e5d891018_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124399704b707f446abb13c112e5d891018_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:25,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/0e9042b2c889404fa3e05c2e16ee198d, store: [table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:25,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/0e9042b2c889404fa3e05c2e16ee198d is 175, key is test_row_0/A:col10/1732481303983/Put/seqid=0 2024-11-24T20:48:25,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742203_1379 (size=31205) 2024-11-24T20:48:25,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:25,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481365485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:25,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:25,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481365485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:25,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:25,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:25,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:25,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481365485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:25,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:25,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:25,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481365485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:25,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:25,774 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=262, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/0e9042b2c889404fa3e05c2e16ee198d 2024-11-24T20:48:25,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/7b1b2a2ecff0485090190a0375b97e7f is 50, key is test_row_0/B:col10/1732481303983/Put/seqid=0 2024-11-24T20:48:25,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742204_1380 (size=12251) 2024-11-24T20:48:25,792 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/7b1b2a2ecff0485090190a0375b97e7f 2024-11-24T20:48:25,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/d5a6ff4d5813476fafba2991a3a7c7de is 50, key is test_row_0/C:col10/1732481303983/Put/seqid=0 2024-11-24T20:48:25,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:25,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481365795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:25,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:25,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481365795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:25,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:25,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:25,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:25,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481365796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:25,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:25,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742205_1381 (size=12251) 2024-11-24T20:48:25,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:25,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481365806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:25,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:25,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:25,815 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/d5a6ff4d5813476fafba2991a3a7c7de 2024-11-24T20:48:25,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/0e9042b2c889404fa3e05c2e16ee198d as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/0e9042b2c889404fa3e05c2e16ee198d 2024-11-24T20:48:25,829 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/0e9042b2c889404fa3e05c2e16ee198d, entries=150, sequenceid=262, filesize=30.5 K 2024-11-24T20:48:25,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/7b1b2a2ecff0485090190a0375b97e7f as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/7b1b2a2ecff0485090190a0375b97e7f 2024-11-24T20:48:25,833 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/7b1b2a2ecff0485090190a0375b97e7f, entries=150, sequenceid=262, filesize=12.0 K 2024-11-24T20:48:25,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/d5a6ff4d5813476fafba2991a3a7c7de as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/d5a6ff4d5813476fafba2991a3a7c7de 2024-11-24T20:48:25,842 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/d5a6ff4d5813476fafba2991a3a7c7de, entries=150, sequenceid=262, filesize=12.0 K 2024-11-24T20:48:25,844 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 72252cc9a6c51612da375fd8db66c697 in 942ms, sequenceid=262, compaction requested=true 2024-11-24T20:48:25,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:25,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:25,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-24T20:48:25,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-24T20:48:25,848 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-24T20:48:25,848 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7100 sec 2024-11-24T20:48:25,853 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.7150 sec 2024-11-24T20:48:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:26,189 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:48:26,190 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72252cc9a6c51612da375fd8db66c697 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-24T20:48:26,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d15a38a3e1c84c8d916a143c1de3ce32_72252cc9a6c51612da375fd8db66c697 is 50, key is test_row_0/A:col10/1732481305152/Put/seqid=0 2024-11-24T20:48:26,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742206_1382 (size=14994) 2024-11-24T20:48:26,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481366246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:26,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:26,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:26,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T20:48:26,286 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:48:26,286 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:48:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-24T20:48:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T20:48:26,288 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:48:26,289 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:48:26,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:48:26,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481366305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:26,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:26,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:26,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:26,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481366315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:26,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:26,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:26,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:26,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481366315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:26,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:26,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:26,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:26,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481366325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:26,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481366365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:26,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:26,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:26,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T20:48:26,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T20:48:26,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:26,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:26,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:26,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:26,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:26,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481366576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:26,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:26,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:26,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:26,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T20:48:26,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:26,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:26,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:26,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:26,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:26,600 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:26,604 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d15a38a3e1c84c8d916a143c1de3ce32_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d15a38a3e1c84c8d916a143c1de3ce32_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T20:48:26,609 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/78539ff575684c799a0ae4666b52c26c, store: [table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:26,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/78539ff575684c799a0ae4666b52c26c is 175, key is test_row_0/A:col10/1732481305152/Put/seqid=0 2024-11-24T20:48:26,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742207_1383 (size=39949) 2024-11-24T20:48:26,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T20:48:26,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:26,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:26,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:26,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:26,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:26,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T20:48:26,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:26,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:26,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:26,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:26,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:26,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:26,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481366895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:26,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:26,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:26,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T20:48:27,019 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/78539ff575684c799a0ae4666b52c26c 2024-11-24T20:48:27,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/8034f1a1519c4db1bb078948a9da3b92 is 50, key is test_row_0/B:col10/1732481305152/Put/seqid=0 2024-11-24T20:48:27,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742208_1384 (size=12301) 2024-11-24T20:48:27,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T20:48:27,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:27,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:27,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:27,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:27,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:27,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:27,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T20:48:27,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:27,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:27,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:27,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:27,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:27,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:27,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481367315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:27,325 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_0' in 72252cc9a6c51612da375fd8db66c697 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:48:27,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:27,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:27,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:27,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481367335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:27,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481367335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:27,341 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_1' in 72252cc9a6c51612da375fd8db66c697 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:48:27,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:27,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:27,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:27,342 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_1' in 72252cc9a6c51612da375fd8db66c697 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:48:27,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:27,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:27,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:27,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481367345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:27,354 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_2' in 72252cc9a6c51612da375fd8db66c697 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:48:27,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:27,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:27,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:27,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T20:48:27,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:27,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:27,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:27,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:27,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:27,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:27,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:27,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481367415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:27,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:27,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:27,424 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:27,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T20:48:27,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/8034f1a1519c4db1bb078948a9da3b92 2024-11-24T20:48:27,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/633747c9052b4023a0d29268a9c3f32d is 50, key is test_row_0/C:col10/1732481305152/Put/seqid=0 2024-11-24T20:48:27,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742209_1385 (size=12301) 2024-11-24T20:48:27,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T20:48:27,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:27,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:27,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:27,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:27,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:27,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T20:48:27,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:27,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:27,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:27,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:27,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:27,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:27,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T20:48:27,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:27,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:27,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:27,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:27,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:27,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:27,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/633747c9052b4023a0d29268a9c3f32d 2024-11-24T20:48:27,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/78539ff575684c799a0ae4666b52c26c as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/78539ff575684c799a0ae4666b52c26c 2024-11-24T20:48:27,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/78539ff575684c799a0ae4666b52c26c, entries=200, sequenceid=287, filesize=39.0 K 2024-11-24T20:48:27,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/8034f1a1519c4db1bb078948a9da3b92 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/8034f1a1519c4db1bb078948a9da3b92 2024-11-24T20:48:27,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/8034f1a1519c4db1bb078948a9da3b92, entries=150, sequenceid=287, filesize=12.0 K 2024-11-24T20:48:27,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/633747c9052b4023a0d29268a9c3f32d as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/633747c9052b4023a0d29268a9c3f32d 2024-11-24T20:48:27,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/633747c9052b4023a0d29268a9c3f32d, entries=150, sequenceid=287, filesize=12.0 K 2024-11-24T20:48:27,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 72252cc9a6c51612da375fd8db66c697 in 1684ms, sequenceid=287, compaction requested=true 2024-11-24T20:48:27,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:27,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72252cc9a6c51612da375fd8db66c697:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:48:27,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:27,873 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:48:27,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72252cc9a6c51612da375fd8db66c697:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:48:27,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:27,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72252cc9a6c51612da375fd8db66c697:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:48:27,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T20:48:27,874 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142554 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:48:27,874 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 72252cc9a6c51612da375fd8db66c697/A is initiating minor compaction (all files) 2024-11-24T20:48:27,874 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72252cc9a6c51612da375fd8db66c697/A in TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:27,874 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/3d7f27e014d14edda6ea103a7d3d761d, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/c16f4d08b8ea4ed3a4719c6abdeb3c0b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/0e9042b2c889404fa3e05c2e16ee198d, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/78539ff575684c799a0ae4666b52c26c] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp, totalSize=139.2 K 2024-11-24T20:48:27,875 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:27,875 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. files: [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/3d7f27e014d14edda6ea103a7d3d761d, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/c16f4d08b8ea4ed3a4719c6abdeb3c0b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/0e9042b2c889404fa3e05c2e16ee198d, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/78539ff575684c799a0ae4666b52c26c] 2024-11-24T20:48:27,876 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d7f27e014d14edda6ea103a7d3d761d, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732481302776 2024-11-24T20:48:27,877 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting c16f4d08b8ea4ed3a4719c6abdeb3c0b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732481303588 2024-11-24T20:48:27,877 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e9042b2c889404fa3e05c2e16ee198d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732481303983 2024-11-24T20:48:27,878 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78539ff575684c799a0ae4666b52c26c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732481305142 2024-11-24T20:48:27,883 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:48:27,886 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49400 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:48:27,886 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 72252cc9a6c51612da375fd8db66c697/B is initiating minor compaction (all files) 2024-11-24T20:48:27,886 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72252cc9a6c51612da375fd8db66c697/B in TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:27,886 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/dacb89c276fb4f47a5b4c9820ba0143c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/628397602fbb46039e09a05b1c118110, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/7b1b2a2ecff0485090190a0375b97e7f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/8034f1a1519c4db1bb078948a9da3b92] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp, totalSize=48.2 K 2024-11-24T20:48:27,888 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting dacb89c276fb4f47a5b4c9820ba0143c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732481302776 2024-11-24T20:48:27,889 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 628397602fbb46039e09a05b1c118110, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732481303588 2024-11-24T20:48:27,889 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b1b2a2ecff0485090190a0375b97e7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732481303983 2024-11-24T20:48:27,890 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 8034f1a1519c4db1bb078948a9da3b92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732481305152 2024-11-24T20:48:27,892 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:27,896 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411248f5f201f00ba452eb38c93d3f0a109ec_72252cc9a6c51612da375fd8db66c697 store=[table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:27,898 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411248f5f201f00ba452eb38c93d3f0a109ec_72252cc9a6c51612da375fd8db66c697, store=[table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:27,899 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411248f5f201f00ba452eb38c93d3f0a109ec_72252cc9a6c51612da375fd8db66c697 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:27,910 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72252cc9a6c51612da375fd8db66c697#B#compaction#324 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:27,911 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/4b84b9605bea4a5d9870c53a9bfa746e is 50, key is test_row_0/B:col10/1732481305152/Put/seqid=0 2024-11-24T20:48:27,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742210_1386 (size=4469) 2024-11-24T20:48:27,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742211_1387 (size=12439) 2024-11-24T20:48:27,933 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/4b84b9605bea4a5d9870c53a9bfa746e as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/4b84b9605bea4a5d9870c53a9bfa746e 2024-11-24T20:48:27,939 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 72252cc9a6c51612da375fd8db66c697/B of 72252cc9a6c51612da375fd8db66c697 into 4b84b9605bea4a5d9870c53a9bfa746e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:27,939 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:27,939 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., storeName=72252cc9a6c51612da375fd8db66c697/B, priority=12, startTime=1732481307873; duration=0sec 2024-11-24T20:48:27,939 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:27,939 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72252cc9a6c51612da375fd8db66c697:B 2024-11-24T20:48:27,940 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:48:27,943 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49400 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:48:27,943 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 72252cc9a6c51612da375fd8db66c697/C is initiating minor compaction (all files) 2024-11-24T20:48:27,943 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72252cc9a6c51612da375fd8db66c697/C in TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:27,943 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/fb004d88026c456597efd2004b35fb84, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/cbbc46cfac2c404da93057d9530a9cd1, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/d5a6ff4d5813476fafba2991a3a7c7de, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/633747c9052b4023a0d29268a9c3f32d] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp, totalSize=48.2 K 2024-11-24T20:48:27,943 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting fb004d88026c456597efd2004b35fb84, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732481302776 2024-11-24T20:48:27,944 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting cbbc46cfac2c404da93057d9530a9cd1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732481303588 2024-11-24T20:48:27,945 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting d5a6ff4d5813476fafba2991a3a7c7de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732481303983 2024-11-24T20:48:27,946 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 633747c9052b4023a0d29268a9c3f32d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732481305152 2024-11-24T20:48:27,958 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72252cc9a6c51612da375fd8db66c697#C#compaction#325 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:27,959 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/f7f3e0b7b9cb4374909978424622edf7 is 50, key is test_row_0/C:col10/1732481305152/Put/seqid=0 2024-11-24T20:48:27,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T20:48:27,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:27,972 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 72252cc9a6c51612da375fd8db66c697 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T20:48:27,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742212_1388 (size=12439) 2024-11-24T20:48:27,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112442a303b347c34a2ea6990237a9b53e55_72252cc9a6c51612da375fd8db66c697 is 50, key is test_row_0/A:col10/1732481306245/Put/seqid=0 2024-11-24T20:48:27,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742213_1389 (size=12454) 2024-11-24T20:48:28,326 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72252cc9a6c51612da375fd8db66c697#A#compaction#323 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:28,327 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/9334523016d54251ac2cde21c1bd4ea7 is 175, key is test_row_0/A:col10/1732481305152/Put/seqid=0 2024-11-24T20:48:28,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742214_1390 (size=31393) 2024-11-24T20:48:28,390 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/f7f3e0b7b9cb4374909978424622edf7 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/f7f3e0b7b9cb4374909978424622edf7 2024-11-24T20:48:28,397 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 72252cc9a6c51612da375fd8db66c697/C of 72252cc9a6c51612da375fd8db66c697 into f7f3e0b7b9cb4374909978424622edf7(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:28,397 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:28,397 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., storeName=72252cc9a6c51612da375fd8db66c697/C, priority=12, startTime=1732481307873; duration=0sec 2024-11-24T20:48:28,397 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:28,397 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72252cc9a6c51612da375fd8db66c697:C 2024-11-24T20:48:28,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:28,404 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112442a303b347c34a2ea6990237a9b53e55_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112442a303b347c34a2ea6990237a9b53e55_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:28,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/fa610eb1ad9245aaae0c962665da6ec5, store: [table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:28,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/fa610eb1ad9245aaae0c962665da6ec5 is 175, key is test_row_0/A:col10/1732481306245/Put/seqid=0 2024-11-24T20:48:28,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742215_1391 (size=31255) 2024-11-24T20:48:28,424 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=299, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/fa610eb1ad9245aaae0c962665da6ec5 2024-11-24T20:48:28,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/f520a554e7244690b3765c33949a33dd is 50, key is test_row_0/B:col10/1732481306245/Put/seqid=0 2024-11-24T20:48:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T20:48:28,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742216_1392 (size=12301) 2024-11-24T20:48:28,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:28,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. as already flushing 2024-11-24T20:48:28,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481368619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:28,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:28,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:28,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:28,735 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/9334523016d54251ac2cde21c1bd4ea7 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/9334523016d54251ac2cde21c1bd4ea7 2024-11-24T20:48:28,738 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 72252cc9a6c51612da375fd8db66c697/A of 72252cc9a6c51612da375fd8db66c697 into 9334523016d54251ac2cde21c1bd4ea7(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:28,738 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:28,738 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., storeName=72252cc9a6c51612da375fd8db66c697/A, priority=12, startTime=1732481307873; duration=0sec 2024-11-24T20:48:28,738 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:28,738 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72252cc9a6c51612da375fd8db66c697:A 2024-11-24T20:48:28,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:28,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481368735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:28,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:28,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:28,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:28,839 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/f520a554e7244690b3765c33949a33dd 2024-11-24T20:48:28,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/138a3cccec6e4d97a087cb253f663110 is 50, key is test_row_0/C:col10/1732481306245/Put/seqid=0 2024-11-24T20:48:28,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742217_1393 (size=12301) 2024-11-24T20:48:28,879 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/138a3cccec6e4d97a087cb253f663110 2024-11-24T20:48:28,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/fa610eb1ad9245aaae0c962665da6ec5 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/fa610eb1ad9245aaae0c962665da6ec5 2024-11-24T20:48:28,891 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/fa610eb1ad9245aaae0c962665da6ec5, entries=150, sequenceid=299, filesize=30.5 K 2024-11-24T20:48:28,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/f520a554e7244690b3765c33949a33dd as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/f520a554e7244690b3765c33949a33dd 2024-11-24T20:48:28,895 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/f520a554e7244690b3765c33949a33dd, entries=150, sequenceid=299, filesize=12.0 K 2024-11-24T20:48:28,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/138a3cccec6e4d97a087cb253f663110 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/138a3cccec6e4d97a087cb253f663110 2024-11-24T20:48:28,903 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/138a3cccec6e4d97a087cb253f663110, entries=150, sequenceid=299, filesize=12.0 K 2024-11-24T20:48:28,904 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 72252cc9a6c51612da375fd8db66c697 in 932ms, sequenceid=299, compaction requested=false 2024-11-24T20:48:28,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:28,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:28,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-24T20:48:28,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-24T20:48:28,907 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-24T20:48:28,907 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6160 sec 2024-11-24T20:48:28,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.6220 sec 2024-11-24T20:48:28,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:28,957 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:48:28,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72252cc9a6c51612da375fd8db66c697 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-24T20:48:28,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112477680081f79f44758ca9e6bc7d64f311_72252cc9a6c51612da375fd8db66c697 is 50, key is test_row_0/A:col10/1732481308616/Put/seqid=0 2024-11-24T20:48:28,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742218_1394 (size=14994) 2024-11-24T20:48:29,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:29,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481369004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:29,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:29,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:29,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:29,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481369115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:29,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:29,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:29,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:29,236 INFO [Thread-1434 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1434. 2024-11-24T20:48:29,236 DEBUG [Thread-1434 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:48:29,236 DEBUG [Thread-1434 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:29,236 DEBUG [Thread-1434 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:29,236 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:48:29,238 INFO [Thread-1436 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1436. 2024-11-24T20:48:29,238 DEBUG [Thread-1436 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:48:29,238 DEBUG [Thread-1436 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:29,238 DEBUG [Thread-1436 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:29,238 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:48:29,240 INFO [Thread-1432 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1432. 2024-11-24T20:48:29,240 DEBUG [Thread-1432 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:48:29,240 DEBUG [Thread-1432 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:29,240 DEBUG [Thread-1432 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:29,240 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:48:29,245 INFO [Thread-1430 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1430. 2024-11-24T20:48:29,245 DEBUG [Thread-1430 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:48:29,245 DEBUG [Thread-1430 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:29,245 DEBUG [Thread-1430 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:29,245 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:48:29,246 INFO [Thread-1428 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1428. 2024-11-24T20:48:29,246 DEBUG [Thread-1428 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:48:29,246 DEBUG [Thread-1428 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:29,246 DEBUG [Thread-1428 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:29,246 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:48:29,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481369325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:29,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:29,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:29,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:29,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:29,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40412 deadline: 1732481369345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40438 deadline: 1732481369345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:29,347 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_1' in 72252cc9a6c51612da375fd8db66c697 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:48:29,347 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_0' in 72252cc9a6c51612da375fd8db66c697 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:48:29,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:29,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:29,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:29,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:29,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:29,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:29,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40462 deadline: 1732481369355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:29,356 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_1' in 72252cc9a6c51612da375fd8db66c697 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:48:29,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:29,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:29,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:29,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:29,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1732481369365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:29,366 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_2' in 72252cc9a6c51612da375fd8db66c697 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:48:29,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:29,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:29,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:29,369 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:29,372 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112477680081f79f44758ca9e6bc7d64f311_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112477680081f79f44758ca9e6bc7d64f311_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:29,373 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/aa1e25d656cb4421897403acafd710f7, store: [table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:29,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/aa1e25d656cb4421897403acafd710f7 is 175, key is test_row_0/A:col10/1732481308616/Put/seqid=0 2024-11-24T20:48:29,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742219_1395 (size=39949) 2024-11-24T20:48:29,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:29,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481369635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:29,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:29,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:29,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:29,778 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=327, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/aa1e25d656cb4421897403acafd710f7 2024-11-24T20:48:29,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/d640e4cb9f714058bf6a7b7340328819 is 50, key is test_row_0/B:col10/1732481308616/Put/seqid=0 2024-11-24T20:48:29,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742220_1396 (size=12301) 2024-11-24T20:48:30,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:30,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40454 deadline: 1732481370145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:30,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:30,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:30,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:48:30,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/d640e4cb9f714058bf6a7b7340328819 2024-11-24T20:48:30,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/864c9b0d6c73423d91c108cf7e5026f6 is 50, key is test_row_0/C:col10/1732481308616/Put/seqid=0 2024-11-24T20:48:30,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742221_1397 (size=12301) 2024-11-24T20:48:30,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T20:48:30,445 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:48:30,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/864c9b0d6c73423d91c108cf7e5026f6 2024-11-24T20:48:30,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/aa1e25d656cb4421897403acafd710f7 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/aa1e25d656cb4421897403acafd710f7 2024-11-24T20:48:30,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/aa1e25d656cb4421897403acafd710f7, entries=200, sequenceid=327, filesize=39.0 K 2024-11-24T20:48:30,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/d640e4cb9f714058bf6a7b7340328819 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/d640e4cb9f714058bf6a7b7340328819 2024-11-24T20:48:30,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/d640e4cb9f714058bf6a7b7340328819, entries=150, sequenceid=327, filesize=12.0 K 2024-11-24T20:48:30,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/864c9b0d6c73423d91c108cf7e5026f6 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/864c9b0d6c73423d91c108cf7e5026f6 2024-11-24T20:48:30,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/864c9b0d6c73423d91c108cf7e5026f6, entries=150, sequenceid=327, filesize=12.0 K 2024-11-24T20:48:30,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 72252cc9a6c51612da375fd8db66c697 in 1663ms, sequenceid=327, compaction requested=true 2024-11-24T20:48:30,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:30,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72252cc9a6c51612da375fd8db66c697:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:48:30,621 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:30,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:30,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72252cc9a6c51612da375fd8db66c697:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:48:30,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:30,621 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:30,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72252cc9a6c51612da375fd8db66c697:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:48:30,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:30,622 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102597 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:30,622 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 72252cc9a6c51612da375fd8db66c697/A is initiating minor compaction (all files) 2024-11-24T20:48:30,622 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37041 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:30,622 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72252cc9a6c51612da375fd8db66c697/A in TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:30,622 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 72252cc9a6c51612da375fd8db66c697/B is initiating minor compaction (all files) 2024-11-24T20:48:30,622 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72252cc9a6c51612da375fd8db66c697/B in TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:30,622 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/9334523016d54251ac2cde21c1bd4ea7, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/fa610eb1ad9245aaae0c962665da6ec5, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/aa1e25d656cb4421897403acafd710f7] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp, totalSize=100.2 K 2024-11-24T20:48:30,622 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/4b84b9605bea4a5d9870c53a9bfa746e, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/f520a554e7244690b3765c33949a33dd, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/d640e4cb9f714058bf6a7b7340328819] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp, totalSize=36.2 K 2024-11-24T20:48:30,622 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:30,622 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. files: [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/9334523016d54251ac2cde21c1bd4ea7, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/fa610eb1ad9245aaae0c962665da6ec5, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/aa1e25d656cb4421897403acafd710f7] 2024-11-24T20:48:30,622 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b84b9605bea4a5d9870c53a9bfa746e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732481305152 2024-11-24T20:48:30,622 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9334523016d54251ac2cde21c1bd4ea7, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732481305152 2024-11-24T20:48:30,623 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting f520a554e7244690b3765c33949a33dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732481306214 2024-11-24T20:48:30,623 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa610eb1ad9245aaae0c962665da6ec5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732481306214 2024-11-24T20:48:30,623 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting d640e4cb9f714058bf6a7b7340328819, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732481308601 2024-11-24T20:48:30,623 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa1e25d656cb4421897403acafd710f7, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732481308577 2024-11-24T20:48:30,630 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72252cc9a6c51612da375fd8db66c697#B#compaction#332 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:30,630 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/6206db2631314a8e966a6da697170a78 is 50, key is test_row_0/B:col10/1732481308616/Put/seqid=0 2024-11-24T20:48:30,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742222_1398 (size=12541) 2024-11-24T20:48:30,638 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:30,639 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124d95171f2dae14b728045d3834c60faec_72252cc9a6c51612da375fd8db66c697 store=[table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:30,664 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124d95171f2dae14b728045d3834c60faec_72252cc9a6c51612da375fd8db66c697, store=[table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:30,665 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d95171f2dae14b728045d3834c60faec_72252cc9a6c51612da375fd8db66c697 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:30,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742223_1399 (size=4469) 2024-11-24T20:48:31,041 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/6206db2631314a8e966a6da697170a78 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/6206db2631314a8e966a6da697170a78 2024-11-24T20:48:31,045 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72252cc9a6c51612da375fd8db66c697/B of 72252cc9a6c51612da375fd8db66c697 into 6206db2631314a8e966a6da697170a78(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:31,045 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:31,045 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., storeName=72252cc9a6c51612da375fd8db66c697/B, priority=13, startTime=1732481310621; duration=0sec 2024-11-24T20:48:31,045 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:31,045 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72252cc9a6c51612da375fd8db66c697:B 2024-11-24T20:48:31,045 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:31,046 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37041 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:31,046 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 72252cc9a6c51612da375fd8db66c697/C is initiating minor compaction (all files) 2024-11-24T20:48:31,046 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72252cc9a6c51612da375fd8db66c697/C in TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:31,046 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/f7f3e0b7b9cb4374909978424622edf7, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/138a3cccec6e4d97a087cb253f663110, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/864c9b0d6c73423d91c108cf7e5026f6] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp, totalSize=36.2 K 2024-11-24T20:48:31,046 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting f7f3e0b7b9cb4374909978424622edf7, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732481305152 2024-11-24T20:48:31,047 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 138a3cccec6e4d97a087cb253f663110, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732481306214 2024-11-24T20:48:31,047 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 864c9b0d6c73423d91c108cf7e5026f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732481308601 2024-11-24T20:48:31,054 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72252cc9a6c51612da375fd8db66c697#C#compaction#334 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:31,054 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/ff8a07066f4a4917ae4c04b8666c2d66 is 50, key is test_row_0/C:col10/1732481308616/Put/seqid=0 2024-11-24T20:48:31,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742224_1400 (size=12541) 2024-11-24T20:48:31,069 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72252cc9a6c51612da375fd8db66c697#A#compaction#333 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:31,069 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/9982a1179ba74c07bcaf84877380c98c is 175, key is test_row_0/A:col10/1732481308616/Put/seqid=0 2024-11-24T20:48:31,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742225_1401 (size=31495) 2024-11-24T20:48:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:31,166 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:48:31,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72252cc9a6c51612da375fd8db66c697 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T20:48:31,166 INFO [Thread-1423 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1423. 2024-11-24T20:48:31,166 DEBUG [Thread-1423 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:48:31,167 DEBUG [Thread-1423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:31,167 DEBUG [Thread-1423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:31,167 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:48:31,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243305d741043a4ee3a7bfb3ce85bf2c48_72252cc9a6c51612da375fd8db66c697 is 50, key is test_row_0/A:col10/1732481311165/Put/seqid=0 2024-11-24T20:48:31,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742226_1402 (size=12454) 2024-11-24T20:48:31,465 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/ff8a07066f4a4917ae4c04b8666c2d66 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/ff8a07066f4a4917ae4c04b8666c2d66 2024-11-24T20:48:31,470 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72252cc9a6c51612da375fd8db66c697/C of 72252cc9a6c51612da375fd8db66c697 into ff8a07066f4a4917ae4c04b8666c2d66(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:31,470 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:31,470 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., storeName=72252cc9a6c51612da375fd8db66c697/C, priority=13, startTime=1732481310621; duration=0sec 2024-11-24T20:48:31,470 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:31,470 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72252cc9a6c51612da375fd8db66c697:C 2024-11-24T20:48:31,477 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/9982a1179ba74c07bcaf84877380c98c as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/9982a1179ba74c07bcaf84877380c98c 2024-11-24T20:48:31,481 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72252cc9a6c51612da375fd8db66c697/A of 72252cc9a6c51612da375fd8db66c697 into 9982a1179ba74c07bcaf84877380c98c(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:31,481 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:31,481 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., storeName=72252cc9a6c51612da375fd8db66c697/A, priority=13, startTime=1732481310621; duration=0sec 2024-11-24T20:48:31,481 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:31,481 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72252cc9a6c51612da375fd8db66c697:A 2024-11-24T20:48:31,586 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:31,589 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243305d741043a4ee3a7bfb3ce85bf2c48_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243305d741043a4ee3a7bfb3ce85bf2c48_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:31,590 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/025112c6f49a4e73b5de30882d58f8e5, store: [table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:31,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/025112c6f49a4e73b5de30882d58f8e5 is 175, key is test_row_0/A:col10/1732481311165/Put/seqid=0 2024-11-24T20:48:31,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742227_1403 (size=31255) 2024-11-24T20:48:31,996 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=339, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/025112c6f49a4e73b5de30882d58f8e5 2024-11-24T20:48:32,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/830880d574834cb2bd1f16a971120e0c is 50, key is test_row_0/B:col10/1732481311165/Put/seqid=0 2024-11-24T20:48:32,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742228_1404 (size=12301) 2024-11-24T20:48:32,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/830880d574834cb2bd1f16a971120e0c 2024-11-24T20:48:32,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/f2c013852a37462d8bc1e7d210c9089a is 50, key is test_row_0/C:col10/1732481311165/Put/seqid=0 2024-11-24T20:48:32,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742229_1405 (size=12301) 2024-11-24T20:48:32,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/f2c013852a37462d8bc1e7d210c9089a 2024-11-24T20:48:32,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/025112c6f49a4e73b5de30882d58f8e5 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/025112c6f49a4e73b5de30882d58f8e5 2024-11-24T20:48:32,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/025112c6f49a4e73b5de30882d58f8e5, entries=150, sequenceid=339, filesize=30.5 K 2024-11-24T20:48:32,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/830880d574834cb2bd1f16a971120e0c as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/830880d574834cb2bd1f16a971120e0c 2024-11-24T20:48:32,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/830880d574834cb2bd1f16a971120e0c, entries=150, sequenceid=339, filesize=12.0 K 2024-11-24T20:48:32,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/f2c013852a37462d8bc1e7d210c9089a as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/f2c013852a37462d8bc1e7d210c9089a 2024-11-24T20:48:32,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/f2c013852a37462d8bc1e7d210c9089a, entries=150, sequenceid=339, filesize=12.0 K 2024-11-24T20:48:32,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for 72252cc9a6c51612da375fd8db66c697 in 1682ms, sequenceid=339, compaction requested=false 2024-11-24T20:48:32,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72252cc9a6c51612da375fd8db66c697: 2024-11-24T20:48:33,377 INFO [Thread-1421 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1421. 2024-11-24T20:48:33,377 DEBUG [Thread-1421 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:48:33,377 DEBUG [Thread-1421 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:33,377 DEBUG [Thread-1421 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:33,377 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:48:33,389 INFO [Thread-1419 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1419. 2024-11-24T20:48:33,389 INFO [Thread-1417 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1417. 2024-11-24T20:48:33,389 DEBUG [Thread-1419 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:48:33,389 DEBUG [Thread-1419 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:33,389 DEBUG [Thread-1417 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:48:33,389 DEBUG [Thread-1417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:33,389 DEBUG [Thread-1419 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:33,389 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:48:33,389 DEBUG [Thread-1417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:33,389 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:48:33,398 INFO [Thread-1425 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1425. 2024-11-24T20:48:33,398 DEBUG [Thread-1425 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:48:33,398 DEBUG [Thread-1425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:33,398 DEBUG [Thread-1425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:33,398 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:48:33,398 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T20:48:33,398 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-24T20:48:33,398 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-24T20:48:33,399 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-24T20:48:33,399 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-11-24T20:48:33,399 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-24T20:48:33,399 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T20:48:33,399 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T20:48:33,399 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1408 2024-11-24T20:48:33,399 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4224 rows 2024-11-24T20:48:33,399 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1406 2024-11-24T20:48:33,399 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4217 rows 2024-11-24T20:48:33,399 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1403 2024-11-24T20:48:33,399 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4208 rows 2024-11-24T20:48:33,399 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1402 2024-11-24T20:48:33,399 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4205 rows 2024-11-24T20:48:33,399 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1419 2024-11-24T20:48:33,399 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4251 rows 2024-11-24T20:48:33,399 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T20:48:33,399 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T20:48:33,399 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:33,400 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:33,400 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:48:33,401 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T20:48:33,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T20:48:33,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T20:48:33,406 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732481313406"}]},"ts":"1732481313406"} 2024-11-24T20:48:33,408 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T20:48:33,408 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T20:48:33,409 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T20:48:33,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72252cc9a6c51612da375fd8db66c697, UNASSIGN}] 2024-11-24T20:48:33,412 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72252cc9a6c51612da375fd8db66c697, UNASSIGN 2024-11-24T20:48:33,413 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=72252cc9a6c51612da375fd8db66c697, regionState=CLOSING, regionLocation=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:33,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=72252cc9a6c51612da375fd8db66c697, UNASSIGN because future has completed 2024-11-24T20:48:33,415 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T20:48:33,415 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291}] 2024-11-24T20:48:33,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T20:48:33,567 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(122): Close 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:33,567 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T20:48:33,567 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1722): Closing 72252cc9a6c51612da375fd8db66c697, disabling compactions & flushes 2024-11-24T20:48:33,567 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:33,567 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:33,568 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. after waiting 0 ms 2024-11-24T20:48:33,568 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:33,568 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(2902): Flushing 72252cc9a6c51612da375fd8db66c697 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-24T20:48:33,603 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e02d5ef383a745b8a37ec308e73569b5_72252cc9a6c51612da375fd8db66c697 is 50, key is test_row_0/A:col10/1732481313396/Put/seqid=0 2024-11-24T20:48:33,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742230_1406 (size=12454) 2024-11-24T20:48:33,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T20:48:34,010 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:34,014 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e02d5ef383a745b8a37ec308e73569b5_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e02d5ef383a745b8a37ec308e73569b5_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:34,014 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/ca353e18e1d249e98234dc83997c8de9, store: [table=TestAcidGuarantees family=A region=72252cc9a6c51612da375fd8db66c697] 2024-11-24T20:48:34,015 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/ca353e18e1d249e98234dc83997c8de9 is 175, key is test_row_0/A:col10/1732481313396/Put/seqid=0 2024-11-24T20:48:34,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742231_1407 (size=31255) 2024-11-24T20:48:34,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T20:48:34,420 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=348, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/ca353e18e1d249e98234dc83997c8de9 2024-11-24T20:48:34,434 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/c2bfea38e88546168a6fb3164c7ab44a is 50, key is test_row_0/B:col10/1732481313396/Put/seqid=0 2024-11-24T20:48:34,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742232_1408 (size=12301) 2024-11-24T20:48:34,437 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/c2bfea38e88546168a6fb3164c7ab44a 2024-11-24T20:48:34,443 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/7959b15689bc4c0fb3951e0d7fc914f8 is 50, key is test_row_0/C:col10/1732481313396/Put/seqid=0 2024-11-24T20:48:34,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742233_1409 (size=12301) 2024-11-24T20:48:34,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T20:48:34,848 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/7959b15689bc4c0fb3951e0d7fc914f8 2024-11-24T20:48:34,852 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/A/ca353e18e1d249e98234dc83997c8de9 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/ca353e18e1d249e98234dc83997c8de9 2024-11-24T20:48:34,855 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/ca353e18e1d249e98234dc83997c8de9, entries=150, sequenceid=348, filesize=30.5 K 2024-11-24T20:48:34,855 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/B/c2bfea38e88546168a6fb3164c7ab44a as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/c2bfea38e88546168a6fb3164c7ab44a 2024-11-24T20:48:34,859 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/c2bfea38e88546168a6fb3164c7ab44a, entries=150, sequenceid=348, filesize=12.0 K 2024-11-24T20:48:34,860 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/.tmp/C/7959b15689bc4c0fb3951e0d7fc914f8 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/7959b15689bc4c0fb3951e0d7fc914f8 2024-11-24T20:48:34,863 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/7959b15689bc4c0fb3951e0d7fc914f8, entries=150, sequenceid=348, filesize=12.0 K 2024-11-24T20:48:34,864 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 72252cc9a6c51612da375fd8db66c697 in 1296ms, sequenceid=348, compaction requested=true 2024-11-24T20:48:34,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/3d7f27e014d14edda6ea103a7d3d761d, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/c16f4d08b8ea4ed3a4719c6abdeb3c0b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/0e9042b2c889404fa3e05c2e16ee198d, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/78539ff575684c799a0ae4666b52c26c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/9334523016d54251ac2cde21c1bd4ea7, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/fa610eb1ad9245aaae0c962665da6ec5, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/aa1e25d656cb4421897403acafd710f7] to archive 2024-11-24T20:48:34,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T20:48:34,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/3d7f27e014d14edda6ea103a7d3d761d to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/3d7f27e014d14edda6ea103a7d3d761d 2024-11-24T20:48:34,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/c16f4d08b8ea4ed3a4719c6abdeb3c0b to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/c16f4d08b8ea4ed3a4719c6abdeb3c0b 2024-11-24T20:48:34,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/0e9042b2c889404fa3e05c2e16ee198d to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/0e9042b2c889404fa3e05c2e16ee198d 2024-11-24T20:48:34,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/78539ff575684c799a0ae4666b52c26c to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/78539ff575684c799a0ae4666b52c26c 2024-11-24T20:48:34,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/9334523016d54251ac2cde21c1bd4ea7 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/9334523016d54251ac2cde21c1bd4ea7 2024-11-24T20:48:34,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/fa610eb1ad9245aaae0c962665da6ec5 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/fa610eb1ad9245aaae0c962665da6ec5 2024-11-24T20:48:34,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/aa1e25d656cb4421897403acafd710f7 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/aa1e25d656cb4421897403acafd710f7 2024-11-24T20:48:34,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/dacb89c276fb4f47a5b4c9820ba0143c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/628397602fbb46039e09a05b1c118110, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/7b1b2a2ecff0485090190a0375b97e7f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/4b84b9605bea4a5d9870c53a9bfa746e, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/8034f1a1519c4db1bb078948a9da3b92, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/f520a554e7244690b3765c33949a33dd, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/d640e4cb9f714058bf6a7b7340328819] to archive 2024-11-24T20:48:34,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T20:48:34,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/dacb89c276fb4f47a5b4c9820ba0143c to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/dacb89c276fb4f47a5b4c9820ba0143c 2024-11-24T20:48:34,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/628397602fbb46039e09a05b1c118110 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/628397602fbb46039e09a05b1c118110 2024-11-24T20:48:34,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/7b1b2a2ecff0485090190a0375b97e7f to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/7b1b2a2ecff0485090190a0375b97e7f 2024-11-24T20:48:34,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/4b84b9605bea4a5d9870c53a9bfa746e to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/4b84b9605bea4a5d9870c53a9bfa746e 2024-11-24T20:48:34,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/8034f1a1519c4db1bb078948a9da3b92 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/8034f1a1519c4db1bb078948a9da3b92 2024-11-24T20:48:34,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/f520a554e7244690b3765c33949a33dd to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/f520a554e7244690b3765c33949a33dd 2024-11-24T20:48:34,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/d640e4cb9f714058bf6a7b7340328819 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/d640e4cb9f714058bf6a7b7340328819 2024-11-24T20:48:34,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/fb004d88026c456597efd2004b35fb84, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/cbbc46cfac2c404da93057d9530a9cd1, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/d5a6ff4d5813476fafba2991a3a7c7de, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/f7f3e0b7b9cb4374909978424622edf7, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/633747c9052b4023a0d29268a9c3f32d, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/138a3cccec6e4d97a087cb253f663110, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/864c9b0d6c73423d91c108cf7e5026f6] to archive 2024-11-24T20:48:34,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T20:48:34,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/fb004d88026c456597efd2004b35fb84 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/fb004d88026c456597efd2004b35fb84 2024-11-24T20:48:34,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/cbbc46cfac2c404da93057d9530a9cd1 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/cbbc46cfac2c404da93057d9530a9cd1 2024-11-24T20:48:34,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/d5a6ff4d5813476fafba2991a3a7c7de to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/d5a6ff4d5813476fafba2991a3a7c7de 2024-11-24T20:48:34,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/f7f3e0b7b9cb4374909978424622edf7 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/f7f3e0b7b9cb4374909978424622edf7 2024-11-24T20:48:34,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/633747c9052b4023a0d29268a9c3f32d to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/633747c9052b4023a0d29268a9c3f32d 2024-11-24T20:48:34,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/138a3cccec6e4d97a087cb253f663110 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/138a3cccec6e4d97a087cb253f663110 2024-11-24T20:48:34,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/864c9b0d6c73423d91c108cf7e5026f6 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/864c9b0d6c73423d91c108cf7e5026f6 2024-11-24T20:48:34,891 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/recovered.edits/351.seqid, newMaxSeqId=351, maxSeqId=4 2024-11-24T20:48:34,891 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. 2024-11-24T20:48:34,891 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1676): Region close journal for 72252cc9a6c51612da375fd8db66c697: Waiting for close lock at 1732481313567Running coprocessor pre-close hooks at 1732481313567Disabling compacts and flushes for region at 1732481313567Disabling writes for close at 1732481313568 (+1 ms)Obtaining lock to block concurrent updates at 1732481313568Preparing flush snapshotting stores in 72252cc9a6c51612da375fd8db66c697 at 1732481313568Finished memstore snapshotting TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732481313568Flushing stores of TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697. at 1732481313594 (+26 ms)Flushing 72252cc9a6c51612da375fd8db66c697/A: creating writer at 1732481313594Flushing mob file 72252cc9a6c51612da375fd8db66c697/A: appending metadata at 1732481313603 (+9 ms)Flushing mob file 72252cc9a6c51612da375fd8db66c697/A: closing flushed file at 1732481313603Flushing 72252cc9a6c51612da375fd8db66c697/A: appending metadata at 1732481314015 (+412 ms)Flushing 72252cc9a6c51612da375fd8db66c697/A: closing flushed file at 1732481314015Flushing 72252cc9a6c51612da375fd8db66c697/B: creating writer at 1732481314431 (+416 ms)Flushing 72252cc9a6c51612da375fd8db66c697/B: appending metadata at 1732481314433 (+2 ms)Flushing 72252cc9a6c51612da375fd8db66c697/B: closing flushed file at 1732481314433Flushing 72252cc9a6c51612da375fd8db66c697/C: creating writer at 1732481314440 (+7 ms)Flushing 72252cc9a6c51612da375fd8db66c697/C: appending metadata at 1732481314443 (+3 ms)Flushing 72252cc9a6c51612da375fd8db66c697/C: closing flushed file at 1732481314443Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4563efe5: reopening flushed file at 1732481314851 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26e895fc: reopening flushed file at 1732481314855 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6dc641e1: reopening flushed file at 1732481314859 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 72252cc9a6c51612da375fd8db66c697 in 1296ms, sequenceid=348, compaction requested=true at 1732481314864 (+5 ms)Writing region close event to WAL at 1732481314888 (+24 ms)Running coprocessor post-close hooks at 1732481314891 (+3 ms)Closed at 1732481314891 2024-11-24T20:48:34,893 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(157): Closed 72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:34,893 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=72252cc9a6c51612da375fd8db66c697, regionState=CLOSED 2024-11-24T20:48:34,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 because future has completed 2024-11-24T20:48:34,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-24T20:48:34,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseRegionProcedure 72252cc9a6c51612da375fd8db66c697, server=20b981ef5c27,34625,1732481182291 in 1.4800 sec 2024-11-24T20:48:34,898 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-24T20:48:34,898 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72252cc9a6c51612da375fd8db66c697, UNASSIGN in 1.4860 sec 2024-11-24T20:48:34,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-24T20:48:34,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4890 sec 2024-11-24T20:48:34,901 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732481314900"}]},"ts":"1732481314900"} 2024-11-24T20:48:34,902 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T20:48:34,902 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T20:48:34,903 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5010 sec 2024-11-24T20:48:35,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T20:48:35,565 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:48:35,566 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T20:48:35,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:48:35,568 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:48:35,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-24T20:48:35,569 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=115, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:48:35,572 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:35,575 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A, FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B, FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C, FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/recovered.edits] 2024-11-24T20:48:35,579 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/025112c6f49a4e73b5de30882d58f8e5 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/025112c6f49a4e73b5de30882d58f8e5 2024-11-24T20:48:35,581 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/9982a1179ba74c07bcaf84877380c98c to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/9982a1179ba74c07bcaf84877380c98c 2024-11-24T20:48:35,582 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/ca353e18e1d249e98234dc83997c8de9 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/A/ca353e18e1d249e98234dc83997c8de9 2024-11-24T20:48:35,584 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/6206db2631314a8e966a6da697170a78 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/6206db2631314a8e966a6da697170a78 2024-11-24T20:48:35,585 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/830880d574834cb2bd1f16a971120e0c to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/830880d574834cb2bd1f16a971120e0c 2024-11-24T20:48:35,586 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/c2bfea38e88546168a6fb3164c7ab44a to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/B/c2bfea38e88546168a6fb3164c7ab44a 2024-11-24T20:48:35,588 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/7959b15689bc4c0fb3951e0d7fc914f8 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/7959b15689bc4c0fb3951e0d7fc914f8 2024-11-24T20:48:35,588 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/f2c013852a37462d8bc1e7d210c9089a to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/f2c013852a37462d8bc1e7d210c9089a 2024-11-24T20:48:35,589 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/ff8a07066f4a4917ae4c04b8666c2d66 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/C/ff8a07066f4a4917ae4c04b8666c2d66 2024-11-24T20:48:35,591 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/recovered.edits/351.seqid to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697/recovered.edits/351.seqid 2024-11-24T20:48:35,592 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:35,592 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T20:48:35,592 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T20:48:35,593 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-24T20:48:35,596 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112400b712bd490f4bc3b24b7eb4e21311f5_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112400b712bd490f4bc3b24b7eb4e21311f5_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:35,597 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243305d741043a4ee3a7bfb3ce85bf2c48_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243305d741043a4ee3a7bfb3ce85bf2c48_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:35,598 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124399704b707f446abb13c112e5d891018_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124399704b707f446abb13c112e5d891018_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:35,599 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243f597db534134416878b962dbf3a78ef_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243f597db534134416878b962dbf3a78ef_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:35,600 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112442a303b347c34a2ea6990237a9b53e55_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112442a303b347c34a2ea6990237a9b53e55_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:35,601 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244658ead7a82742ecb87b6a6b1bd5973e_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244658ead7a82742ecb87b6a6b1bd5973e_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:35,602 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246ff5ccd295964929b61164f7b289cd89_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246ff5ccd295964929b61164f7b289cd89_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:35,603 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247439300b46aa48d39337c2b4054ef47d_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247439300b46aa48d39337c2b4054ef47d_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:35,605 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112477680081f79f44758ca9e6bc7d64f311_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112477680081f79f44758ca9e6bc7d64f311_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:35,606 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112482b9d14110b6472d9c7d08566a66f590_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112482b9d14110b6472d9c7d08566a66f590_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:35,607 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112489130d66d7904644b6ab4700a25cafc7_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112489130d66d7904644b6ab4700a25cafc7_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:35,608 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248ff4ea689187417a932e5a2cf980d43d_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248ff4ea689187417a932e5a2cf980d43d_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:35,610 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249155b63017dd4f978da6e195f241a302_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249155b63017dd4f978da6e195f241a302_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:35,611 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124a02b0b7c01664f1dafb8e1f3d6a4c544_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124a02b0b7c01664f1dafb8e1f3d6a4c544_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:35,612 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b9c564fa304c4038bf7036d603c662a9_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b9c564fa304c4038bf7036d603c662a9_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:35,613 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d15a38a3e1c84c8d916a143c1de3ce32_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d15a38a3e1c84c8d916a143c1de3ce32_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:35,613 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124debb5068d77d493da1c172c0d145db64_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124debb5068d77d493da1c172c0d145db64_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:35,614 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e02d5ef383a745b8a37ec308e73569b5_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e02d5ef383a745b8a37ec308e73569b5_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:35,615 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e14e521bab114a48b286e8642ae9d3ac_72252cc9a6c51612da375fd8db66c697 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e14e521bab114a48b286e8642ae9d3ac_72252cc9a6c51612da375fd8db66c697 2024-11-24T20:48:35,615 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T20:48:35,616 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=115, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:48:35,618 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T20:48:35,620 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T20:48:35,620 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=115, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:48:35,621 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T20:48:35,621 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732481315621"}]},"ts":"9223372036854775807"} 2024-11-24T20:48:35,623 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T20:48:35,623 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 72252cc9a6c51612da375fd8db66c697, NAME => 'TestAcidGuarantees,,1732481286065.72252cc9a6c51612da375fd8db66c697.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T20:48:35,623 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T20:48:35,623 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732481315623"}]},"ts":"9223372036854775807"} 2024-11-24T20:48:35,625 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T20:48:35,625 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=115, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:48:35,626 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 59 msec 2024-11-24T20:48:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-24T20:48:35,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T20:48:35,678 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:48:35,687 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=242 (was 237) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639242380_22 at /127.0.0.1:46236 [Waiting for operation #389] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1639242380_22 at /127.0.0.1:46228 [Waiting for operation #375] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1712698787_22 at /127.0.0.1:33516 [Waiting for operation #541] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/20b981ef5c27:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1712698787_22 at /127.0.0.1:46238 [Waiting for operation #377] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=487 (was 426) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9588 (was 9254) - AvailableMemoryMB LEAK? - 2024-11-24T20:48:35,694 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=242, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=487, ProcessCount=11, AvailableMemoryMB=9587 2024-11-24T20:48:35,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T20:48:35,695 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T20:48:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T20:48:35,697 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T20:48:35,697 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:35,697 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 116 2024-11-24T20:48:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-24T20:48:35,698 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T20:48:35,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742234_1410 (size=959) 2024-11-24T20:48:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-24T20:48:36,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-24T20:48:36,105 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a 2024-11-24T20:48:36,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742235_1411 (size=53) 2024-11-24T20:48:36,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-24T20:48:36,510 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:48:36,510 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 94ec587900a8a4e84a03f1b687bbae49, disabling compactions & flushes 2024-11-24T20:48:36,510 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:36,510 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:36,510 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. after waiting 0 ms 2024-11-24T20:48:36,510 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:36,510 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:36,510 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 94ec587900a8a4e84a03f1b687bbae49: Waiting for close lock at 1732481316510Disabling compacts and flushes for region at 1732481316510Disabling writes for close at 1732481316510Writing region close event to WAL at 1732481316510Closed at 1732481316510 2024-11-24T20:48:36,511 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T20:48:36,512 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732481316511"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732481316511"}]},"ts":"1732481316511"} 2024-11-24T20:48:36,513 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T20:48:36,515 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T20:48:36,515 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732481316515"}]},"ts":"1732481316515"} 2024-11-24T20:48:36,516 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T20:48:36,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94ec587900a8a4e84a03f1b687bbae49, ASSIGN}] 2024-11-24T20:48:36,518 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94ec587900a8a4e84a03f1b687bbae49, ASSIGN 2024-11-24T20:48:36,519 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=94ec587900a8a4e84a03f1b687bbae49, ASSIGN; state=OFFLINE, location=20b981ef5c27,34625,1732481182291; forceNewPlan=false, retain=false 2024-11-24T20:48:36,670 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=94ec587900a8a4e84a03f1b687bbae49, regionState=OPENING, regionLocation=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:36,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=94ec587900a8a4e84a03f1b687bbae49, ASSIGN because future has completed 2024-11-24T20:48:36,672 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291}] 2024-11-24T20:48:36,826 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:36,827 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7752): Opening region: {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} 2024-11-24T20:48:36,828 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:36,828 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:48:36,828 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7794): checking encryption for 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:36,828 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7797): checking classloading for 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:36,830 INFO [StoreOpener-94ec587900a8a4e84a03f1b687bbae49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:36,831 INFO [StoreOpener-94ec587900a8a4e84a03f1b687bbae49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94ec587900a8a4e84a03f1b687bbae49 columnFamilyName A 2024-11-24T20:48:36,831 DEBUG [StoreOpener-94ec587900a8a4e84a03f1b687bbae49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:36,832 INFO [StoreOpener-94ec587900a8a4e84a03f1b687bbae49-1 {}] regionserver.HStore(327): Store=94ec587900a8a4e84a03f1b687bbae49/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:48:36,832 INFO [StoreOpener-94ec587900a8a4e84a03f1b687bbae49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:36,834 INFO [StoreOpener-94ec587900a8a4e84a03f1b687bbae49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94ec587900a8a4e84a03f1b687bbae49 columnFamilyName B 2024-11-24T20:48:36,834 DEBUG [StoreOpener-94ec587900a8a4e84a03f1b687bbae49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:36,834 INFO [StoreOpener-94ec587900a8a4e84a03f1b687bbae49-1 {}] regionserver.HStore(327): Store=94ec587900a8a4e84a03f1b687bbae49/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:48:36,834 INFO [StoreOpener-94ec587900a8a4e84a03f1b687bbae49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:36,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-24T20:48:36,836 INFO [StoreOpener-94ec587900a8a4e84a03f1b687bbae49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94ec587900a8a4e84a03f1b687bbae49 columnFamilyName C 2024-11-24T20:48:36,836 DEBUG [StoreOpener-94ec587900a8a4e84a03f1b687bbae49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:48:36,836 INFO [StoreOpener-94ec587900a8a4e84a03f1b687bbae49-1 {}] regionserver.HStore(327): Store=94ec587900a8a4e84a03f1b687bbae49/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:48:36,836 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1038): replaying wal for 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:36,837 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:36,837 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:36,838 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1048): stopping wal replay for 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:36,838 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1060): Cleaning up temporary data for 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:36,839 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T20:48:36,840 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1093): writing seq id for 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:36,842 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T20:48:36,842 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1114): Opened 94ec587900a8a4e84a03f1b687bbae49; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71953795, jitterRate=0.07219509780406952}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T20:48:36,842 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:36,843 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1006): Region open journal for 94ec587900a8a4e84a03f1b687bbae49: Running coprocessor pre-open hook at 1732481316829Writing region info on filesystem at 1732481316829Initializing all the Stores at 1732481316829Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481316829Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481316830 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481316830Cleaning up temporary data from old regions at 1732481316838 (+8 ms)Running coprocessor post-open hooks at 1732481316842 (+4 ms)Region opened successfully at 1732481316843 (+1 ms) 2024-11-24T20:48:36,844 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., pid=118, masterSystemTime=1732481316824 2024-11-24T20:48:36,846 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:36,847 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:36,847 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=94ec587900a8a4e84a03f1b687bbae49, regionState=OPEN, openSeqNum=2, regionLocation=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:36,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 because future has completed 2024-11-24T20:48:36,852 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-24T20:48:36,852 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; OpenRegionProcedure 94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 in 178 msec 2024-11-24T20:48:36,854 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-24T20:48:36,854 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94ec587900a8a4e84a03f1b687bbae49, ASSIGN in 335 msec 2024-11-24T20:48:36,854 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T20:48:36,854 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732481316854"}]},"ts":"1732481316854"} 2024-11-24T20:48:36,856 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T20:48:36,857 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T20:48:36,859 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1620 sec 2024-11-24T20:48:37,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-24T20:48:37,846 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:48:37,847 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a6d3d4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,847 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:48:37,847 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:48:37,848 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:48:37,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:48:37,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:48:37,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61ef608b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:48:37,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:48:37,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:37,851 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59878, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:48:37,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7138cbd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,852 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:48:37,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:48:37,853 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:48:37,853 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:48:37,854 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T20:48:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 20b981ef5c27,38339,1732481181363 2024-11-24T20:48:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3897c875 2024-11-24T20:48:37,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T20:48:37,856 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59892, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T20:48:37,857 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49f704f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,857 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:48:37,857 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:48:37,858 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:48:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:48:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:48:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38dc0cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:48:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:48:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:37,859 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59912, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:48:37,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33b2c3d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40b7894c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,860 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:48:37,860 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:48:37,861 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:48:37,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:48:37,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:48:37,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d57a15c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:48:37,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:48:37,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:37,862 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:48:37,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ecd910e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,863 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1447b9d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,863 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:48:37,863 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:48:37,864 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:48:37,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:48:37,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:48:37,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33bcaf97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:48:37,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:48:37,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:37,865 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59942, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:48:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d56c418, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,866 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1de6e594, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,866 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:48:37,866 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:48:37,867 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:48:37,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:48:37,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:48:37,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@463d6d48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:48:37,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:48:37,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:37,868 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59962, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:48:37,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3446e284, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,869 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29cadc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,869 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:48:37,869 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:48:37,870 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:48:37,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:48:37,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:48:37,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48d91c0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:48:37,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:48:37,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:37,871 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59974, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:48:37,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dd76e28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,872 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b469189, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,872 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:48:37,872 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:48:37,873 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:48:37,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:48:37,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:48:37,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d83ed7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:48:37,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:48:37,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:37,874 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:48:37,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ee190b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,875 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6240d801, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,875 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:48:37,875 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:48:37,876 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:48:37,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:48:37,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:48:37,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bef0a78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:48:37,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:48:37,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:37,878 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:48:37,878 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12a07cee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,879 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59e2d9ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,879 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:48:37,879 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:48:37,880 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:48:37,880 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:48:37,880 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:48:37,880 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ce7f016, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,880 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:48:37,880 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:48:37,881 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:37,881 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:48:37,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54e2a1dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8d9f546, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,885 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:48:37,885 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:48:37,886 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:48:37,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:48:37,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:48:37,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34abb6f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:48:37,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:48:37,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:37,887 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60032, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:48:37,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a0748c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,889 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45d01c47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,889 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:48:37,889 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:48:37,890 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:48:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:48:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:48:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cb696fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:48:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:48:37,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:37,891 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:48:37,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@334d1368, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:48:37,893 DEBUG [Thread-1886 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:48:37,893 DEBUG [Thread-1871 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:48:37,893 DEBUG [Thread-1882 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:48:37,893 DEBUG [Thread-1879 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:48:37,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:48:37,893 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:48:37,893 DEBUG [Thread-1873 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:48:37,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:48:37,894 DEBUG [Thread-1888 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:48:37,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:48:37,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-24T20:48:37,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:48:37,894 DEBUG [Thread-1875 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:48:37,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:48:37,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:48:37,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:48:37,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:48:37,895 DEBUG [Thread-1877 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:48:37,895 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:48:37,895 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:48:37,895 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51174, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:48:37,895 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:48:37,896 DEBUG [Thread-1875 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:48:37,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:48:37,896 DEBUG [Thread-1875 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:48:37,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:48:37,896 DEBUG [Thread-1877 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:48:37,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-24T20:48:37,896 DEBUG [Thread-1877 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:48:37,897 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:48:37,897 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:48:37,897 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:48:37,897 DEBUG [Thread-1879 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:48:37,897 DEBUG [Thread-1879 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:48:37,897 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:48:37,898 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:48:37,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:48:37,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2] 2024-11-24T20:48:37,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2] 2024-11-24T20:48:37,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2] 2024-11-24T20:48:37,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2] 2024-11-24T20:48:37,900 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:48:37,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2] 2024-11-24T20:48:37,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2] 2024-11-24T20:48:37,901 DEBUG [Thread-1884 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:48:37,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2] 2024-11-24T20:48:37,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2] 2024-11-24T20:48:37,903 DEBUG [Thread-1890 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:48:37,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:48:37,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:48:37,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:48:37,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:48:37,905 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:48:37,905 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:48:37,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2] 2024-11-24T20:48:37,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2] 2024-11-24T20:48:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:37,962 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:48:37,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94ec587900a8a4e84a03f1b687bbae49 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-24T20:48:37,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/9d7c0cefd6064654a44eb23671aaff9d is 50, key is test_row_0/A:col10/1732481317924/Put/seqid=0 2024-11-24T20:48:38,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-24T20:48:38,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742236_1412 (size=14341) 2024-11-24T20:48:38,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/9d7c0cefd6064654a44eb23671aaff9d 2024-11-24T20:48:38,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:38,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481378006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:38,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:38,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51214 deadline: 1732481378017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:38,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51228 deadline: 1732481378018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:38,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481378019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:38,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51226 deadline: 1732481378019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:38,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-24T20:48:38,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:38,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:38,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/1b067ce9a2f9416b84686b7f7f19857b is 50, key is test_row_0/B:col10/1732481317924/Put/seqid=0 2024-11-24T20:48:38,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:38,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:38,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:38,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:38,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742237_1413 (size=12001) 2024-11-24T20:48:38,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/1b067ce9a2f9416b84686b7f7f19857b 2024-11-24T20:48:38,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/9dde0ffc3f4a4e19938224c3ebe09074 is 50, key is test_row_0/C:col10/1732481317924/Put/seqid=0 2024-11-24T20:48:38,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742238_1414 (size=12001) 2024-11-24T20:48:38,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:38,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481378125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:38,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:38,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/9dde0ffc3f4a4e19938224c3ebe09074 2024-11-24T20:48:38,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/9d7c0cefd6064654a44eb23671aaff9d as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/9d7c0cefd6064654a44eb23671aaff9d 2024-11-24T20:48:38,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:38,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51214 deadline: 1732481378135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:38,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:38,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51228 deadline: 1732481378135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:38,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:38,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51226 deadline: 1732481378135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:38,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481378135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:38,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/9d7c0cefd6064654a44eb23671aaff9d, entries=200, sequenceid=15, filesize=14.0 K 2024-11-24T20:48:38,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/1b067ce9a2f9416b84686b7f7f19857b as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/1b067ce9a2f9416b84686b7f7f19857b 2024-11-24T20:48:38,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/1b067ce9a2f9416b84686b7f7f19857b, entries=150, sequenceid=15, filesize=11.7 K 2024-11-24T20:48:38,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/9dde0ffc3f4a4e19938224c3ebe09074 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/9dde0ffc3f4a4e19938224c3ebe09074 2024-11-24T20:48:38,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/9dde0ffc3f4a4e19938224c3ebe09074, entries=150, sequenceid=15, filesize=11.7 K 2024-11-24T20:48:38,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 94ec587900a8a4e84a03f1b687bbae49 in 202ms, sequenceid=15, compaction requested=false 2024-11-24T20:48:38,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:38,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-24T20:48:38,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:38,208 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 94ec587900a8a4e84a03f1b687bbae49 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-24T20:48:38,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-24T20:48:38,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/535f9ae0b00c4914bc0fc9743c59995f is 50, key is test_row_0/A:col10/1732481318017/Put/seqid=0 2024-11-24T20:48:38,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742239_1415 (size=12001) 2024-11-24T20:48:38,236 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/535f9ae0b00c4914bc0fc9743c59995f 2024-11-24T20:48:38,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/e5dc6681a9574d8b8c443f452b4ed0a6 is 50, key is test_row_0/B:col10/1732481318017/Put/seqid=0 2024-11-24T20:48:38,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742240_1416 (size=12001) 2024-11-24T20:48:38,287 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/e5dc6681a9574d8b8c443f452b4ed0a6 2024-11-24T20:48:38,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/12396d7056534ebd91b747c4a7a539b1 is 50, key is test_row_0/C:col10/1732481318017/Put/seqid=0 2024-11-24T20:48:38,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742241_1417 (size=12001) 2024-11-24T20:48:38,334 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/12396d7056534ebd91b747c4a7a539b1 2024-11-24T20:48:38,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:38,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:38,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/535f9ae0b00c4914bc0fc9743c59995f as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/535f9ae0b00c4914bc0fc9743c59995f 2024-11-24T20:48:38,347 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/535f9ae0b00c4914bc0fc9743c59995f, entries=150, sequenceid=38, filesize=11.7 K 2024-11-24T20:48:38,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/e5dc6681a9574d8b8c443f452b4ed0a6 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/e5dc6681a9574d8b8c443f452b4ed0a6 2024-11-24T20:48:38,363 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/e5dc6681a9574d8b8c443f452b4ed0a6, entries=150, sequenceid=38, filesize=11.7 K 2024-11-24T20:48:38,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/12396d7056534ebd91b747c4a7a539b1 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/12396d7056534ebd91b747c4a7a539b1 2024-11-24T20:48:38,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:38,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481378362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:38,378 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,378 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,378 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:38,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:38,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481378372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:38,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:38,383 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/12396d7056534ebd91b747c4a7a539b1, entries=150, sequenceid=38, filesize=11.7 K 2024-11-24T20:48:38,385 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 94ec587900a8a4e84a03f1b687bbae49 in 177ms, sequenceid=38, compaction requested=false 2024-11-24T20:48:38,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:38,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:38,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-24T20:48:38,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-24T20:48:38,388 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-24T20:48:38,389 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 488 msec 2024-11-24T20:48:38,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:38,389 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:48:38,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94ec587900a8a4e84a03f1b687bbae49 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-24T20:48:38,393 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 496 msec 2024-11-24T20:48:38,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/15f24effc8584d219a5e0a74e6f27540 is 50, key is test_row_0/A:col10/1732481318386/Put/seqid=0 2024-11-24T20:48:38,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742242_1418 (size=14341) 2024-11-24T20:48:38,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/15f24effc8584d219a5e0a74e6f27540 2024-11-24T20:48:38,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/57d71bb319fb4e6ca75ae1b5c778c578 is 50, key is test_row_0/B:col10/1732481318386/Put/seqid=0 2024-11-24T20:48:38,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742243_1419 (size=12001) 2024-11-24T20:48:38,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/57d71bb319fb4e6ca75ae1b5c778c578 2024-11-24T20:48:38,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-24T20:48:38,526 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:48:38,526 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:48:38,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-24T20:48:38,535 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:48:38,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T20:48:38,536 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:48:38,537 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:48:38,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/9adcec31e13a484c9fccfc9959762064 is 50, key is test_row_0/C:col10/1732481318386/Put/seqid=0 2024-11-24T20:48:38,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:38,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481378485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:38,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742244_1420 (size=12001) 2024-11-24T20:48:38,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:38,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:38,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481378485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:38,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:38,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:38,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51214 deadline: 1732481378535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:38,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:38,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:38,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51228 deadline: 1732481378536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:38,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:38,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:38,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51226 deadline: 1732481378536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:38,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:38,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T20:48:38,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51228 deadline: 1732481378655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:38,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51214 deadline: 1732481378655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:38,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51226 deadline: 1732481378655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:38,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-24T20:48:38,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:38,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:38,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:38,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:38,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:38,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:38,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:38,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481378755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:38,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:38,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:38,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481378755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:38,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:38,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-24T20:48:38,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:38,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:38,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:38,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:38,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:38,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:38,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T20:48:38,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:38,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51228 deadline: 1732481378875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:38,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51214 deadline: 1732481378875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:38,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51226 deadline: 1732481378875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:38,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:38,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/9adcec31e13a484c9fccfc9959762064 2024-11-24T20:48:38,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/15f24effc8584d219a5e0a74e6f27540 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/15f24effc8584d219a5e0a74e6f27540 2024-11-24T20:48:38,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/15f24effc8584d219a5e0a74e6f27540, entries=200, sequenceid=52, filesize=14.0 K 2024-11-24T20:48:38,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/57d71bb319fb4e6ca75ae1b5c778c578 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/57d71bb319fb4e6ca75ae1b5c778c578 2024-11-24T20:48:38,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/57d71bb319fb4e6ca75ae1b5c778c578, entries=150, sequenceid=52, filesize=11.7 K 2024-11-24T20:48:38,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/9adcec31e13a484c9fccfc9959762064 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/9adcec31e13a484c9fccfc9959762064 2024-11-24T20:48:38,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/9adcec31e13a484c9fccfc9959762064, entries=150, sequenceid=52, filesize=11.7 K 2024-11-24T20:48:38,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 94ec587900a8a4e84a03f1b687bbae49 in 589ms, sequenceid=52, compaction requested=true 2024-11-24T20:48:38,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:38,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:48:38,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:38,979 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:38,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:48:38,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:38,979 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:38,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:48:38,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:38,980 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:38,980 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:38,980 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/B is initiating minor compaction (all files) 2024-11-24T20:48:38,980 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/A is initiating minor compaction (all files) 2024-11-24T20:48:38,980 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/A in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:38,980 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/B in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:38,980 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/9d7c0cefd6064654a44eb23671aaff9d, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/535f9ae0b00c4914bc0fc9743c59995f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/15f24effc8584d219a5e0a74e6f27540] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=39.7 K 2024-11-24T20:48:38,980 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/1b067ce9a2f9416b84686b7f7f19857b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/e5dc6681a9574d8b8c443f452b4ed0a6, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/57d71bb319fb4e6ca75ae1b5c778c578] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=35.2 K 2024-11-24T20:48:38,980 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b067ce9a2f9416b84686b7f7f19857b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732481317908 2024-11-24T20:48:38,980 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d7c0cefd6064654a44eb23671aaff9d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732481317908 2024-11-24T20:48:38,980 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 535f9ae0b00c4914bc0fc9743c59995f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732481318002 2024-11-24T20:48:38,980 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting e5dc6681a9574d8b8c443f452b4ed0a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732481318002 2024-11-24T20:48:38,981 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 57d71bb319fb4e6ca75ae1b5c778c578, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732481318372 2024-11-24T20:48:38,981 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15f24effc8584d219a5e0a74e6f27540, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732481318360 2024-11-24T20:48:38,990 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#B#compaction#350 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:38,990 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/ef0b517001964fa595d58f26a318dcb9 is 50, key is test_row_0/B:col10/1732481318386/Put/seqid=0 2024-11-24T20:48:38,992 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#A#compaction#351 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:38,992 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/dc78a5ae1aa34e99a9c7f5db591906bb is 50, key is test_row_0/A:col10/1732481318386/Put/seqid=0 2024-11-24T20:48:38,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-24T20:48:38,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:38,996 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 94ec587900a8a4e84a03f1b687bbae49 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-24T20:48:38,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742246_1422 (size=12104) 2024-11-24T20:48:39,010 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/dc78a5ae1aa34e99a9c7f5db591906bb as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/dc78a5ae1aa34e99a9c7f5db591906bb 2024-11-24T20:48:39,017 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/A of 94ec587900a8a4e84a03f1b687bbae49 into dc78a5ae1aa34e99a9c7f5db591906bb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:39,017 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:39,017 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/A, priority=13, startTime=1732481318979; duration=0sec 2024-11-24T20:48:39,017 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:39,017 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:A 2024-11-24T20:48:39,017 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:39,020 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:39,020 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/C is initiating minor compaction (all files) 2024-11-24T20:48:39,020 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/C in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:39,020 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/9dde0ffc3f4a4e19938224c3ebe09074, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/12396d7056534ebd91b747c4a7a539b1, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/9adcec31e13a484c9fccfc9959762064] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=35.2 K 2024-11-24T20:48:39,020 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9dde0ffc3f4a4e19938224c3ebe09074, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732481317908 2024-11-24T20:48:39,021 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12396d7056534ebd91b747c4a7a539b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732481318002 2024-11-24T20:48:39,021 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9adcec31e13a484c9fccfc9959762064, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732481318372 2024-11-24T20:48:39,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742245_1421 (size=12104) 2024-11-24T20:48:39,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/23c4c832b1644fb68a457d6ade83b583 is 50, key is test_row_0/A:col10/1732481318484/Put/seqid=0 2024-11-24T20:48:39,045 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#C#compaction#353 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:39,046 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/75e9286d95e84d209e1d98d0fa753eed is 50, key is test_row_0/C:col10/1732481318386/Put/seqid=0 2024-11-24T20:48:39,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742247_1423 (size=12001) 2024-11-24T20:48:39,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742248_1424 (size=12104) 2024-11-24T20:48:39,059 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/75e9286d95e84d209e1d98d0fa753eed as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/75e9286d95e84d209e1d98d0fa753eed 2024-11-24T20:48:39,063 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/C of 94ec587900a8a4e84a03f1b687bbae49 into 75e9286d95e84d209e1d98d0fa753eed(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:39,063 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:39,063 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/C, priority=13, startTime=1732481318979; duration=0sec 2024-11-24T20:48:39,063 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:39,063 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:C 2024-11-24T20:48:39,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:39,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:39,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:39,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481379115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:39,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:39,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:39,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:39,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:39,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481379123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:39,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:39,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:39,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:39,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T20:48:39,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:39,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51228 deadline: 1732481379185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:39,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:39,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51214 deadline: 1732481379185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:39,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:39,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:39,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51226 deadline: 1732481379185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:39,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:39,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:39,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:39,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:39,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481379235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:39,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:39,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:39,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481379235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:39,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:39,433 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/ef0b517001964fa595d58f26a318dcb9 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/ef0b517001964fa595d58f26a318dcb9 2024-11-24T20:48:39,437 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/B of 94ec587900a8a4e84a03f1b687bbae49 into ef0b517001964fa595d58f26a318dcb9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:39,437 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:39,437 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/B, priority=13, startTime=1732481318979; duration=0sec 2024-11-24T20:48:39,437 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:39,437 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:B 2024-11-24T20:48:39,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:39,449 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/23c4c832b1644fb68a457d6ade83b583 2024-11-24T20:48:39,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481379444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:39,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:39,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:39,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:39,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:39,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481379445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:39,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:39,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:39,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:39,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/a1ca01cf7e8f4ae2a840efb1e85bbe1e is 50, key is test_row_0/B:col10/1732481318484/Put/seqid=0 2024-11-24T20:48:39,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742249_1425 (size=12001) 2024-11-24T20:48:39,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T20:48:39,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:39,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51228 deadline: 1732481379694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:39,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:39,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:39,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:39,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51214 deadline: 1732481379695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:39,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:39,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:39,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:39,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:39,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51226 deadline: 1732481379695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:39,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:39,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:39,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:39,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:39,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:39,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481379755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:39,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:39,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:39,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:39,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:39,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481379755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:39,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:39,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:39,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:39,865 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/a1ca01cf7e8f4ae2a840efb1e85bbe1e 2024-11-24T20:48:39,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/b967c3d465d3417aaa30bf1723cc5b9a is 50, key is test_row_0/C:col10/1732481318484/Put/seqid=0 2024-11-24T20:48:39,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742250_1426 (size=12001) 2024-11-24T20:48:39,882 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/b967c3d465d3417aaa30bf1723cc5b9a 2024-11-24T20:48:39,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/23c4c832b1644fb68a457d6ade83b583 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/23c4c832b1644fb68a457d6ade83b583 2024-11-24T20:48:39,898 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/23c4c832b1644fb68a457d6ade83b583, entries=150, sequenceid=74, filesize=11.7 K 2024-11-24T20:48:39,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/a1ca01cf7e8f4ae2a840efb1e85bbe1e as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/a1ca01cf7e8f4ae2a840efb1e85bbe1e 2024-11-24T20:48:39,905 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/a1ca01cf7e8f4ae2a840efb1e85bbe1e, entries=150, sequenceid=74, filesize=11.7 K 2024-11-24T20:48:39,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/b967c3d465d3417aaa30bf1723cc5b9a as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/b967c3d465d3417aaa30bf1723cc5b9a 2024-11-24T20:48:39,911 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/b967c3d465d3417aaa30bf1723cc5b9a, entries=150, sequenceid=74, filesize=11.7 K 2024-11-24T20:48:39,912 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 94ec587900a8a4e84a03f1b687bbae49 in 916ms, sequenceid=74, compaction requested=false 2024-11-24T20:48:39,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:39,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:39,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-24T20:48:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-24T20:48:39,916 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-24T20:48:39,916 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3770 sec 2024-11-24T20:48:39,917 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.3900 sec 2024-11-24T20:48:40,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:40,270 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:48:40,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94ec587900a8a4e84a03f1b687bbae49 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-24T20:48:40,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/3b48c409a3c64c79988f11094bdd91c0 is 50, key is test_row_0/A:col10/1732481320268/Put/seqid=0 2024-11-24T20:48:40,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742251_1427 (size=16681) 2024-11-24T20:48:40,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/3b48c409a3c64c79988f11094bdd91c0 2024-11-24T20:48:40,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/7f415bccf77b4174be241e7ad0e2d5ff is 50, key is test_row_0/B:col10/1732481320268/Put/seqid=0 2024-11-24T20:48:40,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742252_1428 (size=12001) 2024-11-24T20:48:40,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/7f415bccf77b4174be241e7ad0e2d5ff 2024-11-24T20:48:40,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/12604e9a4c4f4e8996eab86d70d78117 is 50, key is test_row_0/C:col10/1732481320268/Put/seqid=0 2024-11-24T20:48:40,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742253_1429 (size=12001) 2024-11-24T20:48:40,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/12604e9a4c4f4e8996eab86d70d78117 2024-11-24T20:48:40,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/3b48c409a3c64c79988f11094bdd91c0 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/3b48c409a3c64c79988f11094bdd91c0 2024-11-24T20:48:40,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/3b48c409a3c64c79988f11094bdd91c0, entries=250, sequenceid=93, filesize=16.3 K 2024-11-24T20:48:40,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/7f415bccf77b4174be241e7ad0e2d5ff as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/7f415bccf77b4174be241e7ad0e2d5ff 2024-11-24T20:48:40,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/7f415bccf77b4174be241e7ad0e2d5ff, entries=150, sequenceid=93, filesize=11.7 K 2024-11-24T20:48:40,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/12604e9a4c4f4e8996eab86d70d78117 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/12604e9a4c4f4e8996eab86d70d78117 2024-11-24T20:48:40,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/12604e9a4c4f4e8996eab86d70d78117, entries=150, sequenceid=93, filesize=11.7 K 2024-11-24T20:48:40,340 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=93.93 KB/96180 for 94ec587900a8a4e84a03f1b687bbae49 in 70ms, sequenceid=93, compaction requested=true 2024-11-24T20:48:40,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:40,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:48:40,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:40,341 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:40,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:48:40,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:40,341 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:40,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:48:40,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:40,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:40,342 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:48:40,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94ec587900a8a4e84a03f1b687bbae49 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-24T20:48:40,343 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:40,343 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:40,343 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/A is initiating minor compaction (all files) 2024-11-24T20:48:40,343 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/B is initiating minor compaction (all files) 2024-11-24T20:48:40,343 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/A in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:40,344 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/dc78a5ae1aa34e99a9c7f5db591906bb, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/23c4c832b1644fb68a457d6ade83b583, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/3b48c409a3c64c79988f11094bdd91c0] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=39.8 K 2024-11-24T20:48:40,344 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/B in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:40,344 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/ef0b517001964fa595d58f26a318dcb9, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/a1ca01cf7e8f4ae2a840efb1e85bbe1e, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/7f415bccf77b4174be241e7ad0e2d5ff] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=35.3 K 2024-11-24T20:48:40,346 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting ef0b517001964fa595d58f26a318dcb9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732481318372 2024-11-24T20:48:40,346 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc78a5ae1aa34e99a9c7f5db591906bb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732481318372 2024-11-24T20:48:40,346 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting a1ca01cf7e8f4ae2a840efb1e85bbe1e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732481318458 2024-11-24T20:48:40,346 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23c4c832b1644fb68a457d6ade83b583, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732481318458 2024-11-24T20:48:40,346 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f415bccf77b4174be241e7ad0e2d5ff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732481319122 2024-11-24T20:48:40,347 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b48c409a3c64c79988f11094bdd91c0, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732481319103 2024-11-24T20:48:40,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/87657b74b75e4f1c8cce8cd89d38ec94 is 50, key is test_row_0/A:col10/1732481320329/Put/seqid=0 2024-11-24T20:48:40,353 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#B#compaction#361 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:40,353 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#A#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:40,353 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/b10f6d1da31f497a98fd53463cce39d7 is 50, key is test_row_0/B:col10/1732481320268/Put/seqid=0 2024-11-24T20:48:40,354 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/28f34beb11c248ffba737881342d4af1 is 50, key is test_row_0/A:col10/1732481320268/Put/seqid=0 2024-11-24T20:48:40,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742254_1430 (size=16681) 2024-11-24T20:48:40,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742255_1431 (size=12207) 2024-11-24T20:48:40,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/87657b74b75e4f1c8cce8cd89d38ec94 2024-11-24T20:48:40,368 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/b10f6d1da31f497a98fd53463cce39d7 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/b10f6d1da31f497a98fd53463cce39d7 2024-11-24T20:48:40,372 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/B of 94ec587900a8a4e84a03f1b687bbae49 into b10f6d1da31f497a98fd53463cce39d7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:40,372 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:40,372 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/B, priority=13, startTime=1732481320341; duration=0sec 2024-11-24T20:48:40,372 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:40,372 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:B 2024-11-24T20:48:40,372 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:40,374 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:40,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742256_1432 (size=12207) 2024-11-24T20:48:40,374 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/C is initiating minor compaction (all files) 2024-11-24T20:48:40,374 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/C in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:40,374 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/75e9286d95e84d209e1d98d0fa753eed, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/b967c3d465d3417aaa30bf1723cc5b9a, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/12604e9a4c4f4e8996eab86d70d78117] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=35.3 K 2024-11-24T20:48:40,375 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 75e9286d95e84d209e1d98d0fa753eed, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732481318372 2024-11-24T20:48:40,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/a39eb05eedcf42b5a6b82deddb07b700 is 50, key is test_row_0/B:col10/1732481320329/Put/seqid=0 2024-11-24T20:48:40,376 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting b967c3d465d3417aaa30bf1723cc5b9a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732481318458 2024-11-24T20:48:40,377 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 12604e9a4c4f4e8996eab86d70d78117, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732481319122 2024-11-24T20:48:40,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742257_1433 (size=12001) 2024-11-24T20:48:40,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/a39eb05eedcf42b5a6b82deddb07b700 2024-11-24T20:48:40,392 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#C#compaction#363 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:40,392 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/f2a0b69d54ca43959f0f0dc32d9f0186 is 50, key is test_row_0/C:col10/1732481320268/Put/seqid=0 2024-11-24T20:48:40,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742258_1434 (size=12207) 2024-11-24T20:48:40,402 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/f2a0b69d54ca43959f0f0dc32d9f0186 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/f2a0b69d54ca43959f0f0dc32d9f0186 2024-11-24T20:48:40,408 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/C of 94ec587900a8a4e84a03f1b687bbae49 into f2a0b69d54ca43959f0f0dc32d9f0186(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:40,408 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:40,408 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/C, priority=13, startTime=1732481320341; duration=0sec 2024-11-24T20:48:40,408 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:40,408 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:C 2024-11-24T20:48:40,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/7f6207dd546c4e12a2bf7ed757d0c1b9 is 50, key is test_row_0/C:col10/1732481320329/Put/seqid=0 2024-11-24T20:48:40,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742259_1435 (size=12001) 2024-11-24T20:48:40,422 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T20:48:40,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:40,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481380436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:40,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:40,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481380436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:40,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:40,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:40,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:40,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:40,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:40,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:40,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:40,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481380555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:40,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:40,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:40,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:40,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481380555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:40,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:40,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:40,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:40,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:40,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T20:48:40,685 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:48:40,685 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:48:40,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-24T20:48:40,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T20:48:40,686 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:48:40,687 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:48:40,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:48:40,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51228 deadline: 1732481380715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:40,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:40,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51214 deadline: 1732481380715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:40,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:40,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51226 deadline: 1732481380715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:40,721 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_0' in 94ec587900a8a4e84a03f1b687bbae49 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:48:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:40,721 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_0' in 94ec587900a8a4e84a03f1b687bbae49 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:48:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:40,721 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_2' in 94ec587900a8a4e84a03f1b687bbae49 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:48:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:40,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:40,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481380765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:40,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:40,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:40,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:40,778 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/28f34beb11c248ffba737881342d4af1 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/28f34beb11c248ffba737881342d4af1 2024-11-24T20:48:40,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:40,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481380775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:40,781 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/A of 94ec587900a8a4e84a03f1b687bbae49 into 28f34beb11c248ffba737881342d4af1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:40,781 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:40,781 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/A, priority=13, startTime=1732481320340; duration=0sec 2024-11-24T20:48:40,781 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:40,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:40,781 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:A 2024-11-24T20:48:40,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:40,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:40,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T20:48:40,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/7f6207dd546c4e12a2bf7ed757d0c1b9 2024-11-24T20:48:40,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/87657b74b75e4f1c8cce8cd89d38ec94 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/87657b74b75e4f1c8cce8cd89d38ec94 2024-11-24T20:48:40,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/87657b74b75e4f1c8cce8cd89d38ec94, entries=250, sequenceid=112, filesize=16.3 K 2024-11-24T20:48:40,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/a39eb05eedcf42b5a6b82deddb07b700 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/a39eb05eedcf42b5a6b82deddb07b700 2024-11-24T20:48:40,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/a39eb05eedcf42b5a6b82deddb07b700, entries=150, sequenceid=112, filesize=11.7 K 2024-11-24T20:48:40,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/7f6207dd546c4e12a2bf7ed757d0c1b9 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/7f6207dd546c4e12a2bf7ed757d0c1b9 2024-11-24T20:48:40,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/7f6207dd546c4e12a2bf7ed757d0c1b9, entries=150, sequenceid=112, filesize=11.7 K 2024-11-24T20:48:40,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 94ec587900a8a4e84a03f1b687bbae49 in 491ms, sequenceid=112, compaction requested=false 2024-11-24T20:48:40,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:40,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-24T20:48:40,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:40,839 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 94ec587900a8a4e84a03f1b687bbae49 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-24T20:48:40,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/ac65fa7f5bcd4df2a5223744718cf7d4 is 50, key is test_row_0/A:col10/1732481320417/Put/seqid=0 2024-11-24T20:48:40,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742260_1436 (size=12101) 2024-11-24T20:48:41,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T20:48:41,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:41,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:41,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:41,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481381143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:41,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:41,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:41,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:41,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:41,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481381152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:41,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:41,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:41,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:41,252 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/ac65fa7f5bcd4df2a5223744718cf7d4 2024-11-24T20:48:41,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/fb185c86f8f54d6ba26c7207b7b1c21b is 50, key is test_row_0/B:col10/1732481320417/Put/seqid=0 2024-11-24T20:48:41,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:41,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481381255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:41,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:41,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:41,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:41,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742261_1437 (size=12101) 2024-11-24T20:48:41,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:41,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481381265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:41,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T20:48:41,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:41,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481381465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:41,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:41,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:41,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:41,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:41,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481381475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:41,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:41,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:41,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:41,662 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/fb185c86f8f54d6ba26c7207b7b1c21b 2024-11-24T20:48:41,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/dd832261c2864a52b981cca1d7fa9701 is 50, key is test_row_0/C:col10/1732481320417/Put/seqid=0 2024-11-24T20:48:41,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742262_1438 (size=12101) 2024-11-24T20:48:41,671 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/dd832261c2864a52b981cca1d7fa9701 2024-11-24T20:48:41,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/ac65fa7f5bcd4df2a5223744718cf7d4 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/ac65fa7f5bcd4df2a5223744718cf7d4 2024-11-24T20:48:41,677 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/ac65fa7f5bcd4df2a5223744718cf7d4, entries=150, sequenceid=132, filesize=11.8 K 2024-11-24T20:48:41,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/fb185c86f8f54d6ba26c7207b7b1c21b as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/fb185c86f8f54d6ba26c7207b7b1c21b 2024-11-24T20:48:41,681 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/fb185c86f8f54d6ba26c7207b7b1c21b, entries=150, sequenceid=132, filesize=11.8 K 2024-11-24T20:48:41,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/dd832261c2864a52b981cca1d7fa9701 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/dd832261c2864a52b981cca1d7fa9701 2024-11-24T20:48:41,684 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/dd832261c2864a52b981cca1d7fa9701, entries=150, sequenceid=132, filesize=11.8 K 2024-11-24T20:48:41,685 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 94ec587900a8a4e84a03f1b687bbae49 in 846ms, sequenceid=132, compaction requested=true 2024-11-24T20:48:41,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:41,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:41,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-24T20:48:41,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-24T20:48:41,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-24T20:48:41,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 999 msec 2024-11-24T20:48:41,688 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.0020 sec 2024-11-24T20:48:41,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:41,790 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:48:41,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94ec587900a8a4e84a03f1b687bbae49 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-24T20:48:41,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/1231c89e5d6f499a92190db3eea5cbcd is 50, key is test_row_0/A:col10/1732481321789/Put/seqid=0 2024-11-24T20:48:41,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742263_1439 (size=14541) 2024-11-24T20:48:41,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T20:48:41,825 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:48:41,825 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:48:41,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-24T20:48:41,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T20:48:41,827 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:48:41,828 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:48:41,828 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:48:41,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:41,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481381844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:41,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:41,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:41,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:41,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:41,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481381845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:41,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:41,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:41,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T20:48:41,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:41,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481381955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:41,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:41,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481381965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:41,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:41,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:41,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:41,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T20:48:41,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:41,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:41,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:41,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:41,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:42,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T20:48:42,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:42,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:42,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:42,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:42,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:42,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:42,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T20:48:42,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:42,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481382165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:42,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:42,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:42,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:42,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:42,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481382175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:42,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:42,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:42,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:42,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/1231c89e5d6f499a92190db3eea5cbcd 2024-11-24T20:48:42,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/42d92306bf314ff3b6917c7f8255dea2 is 50, key is test_row_0/B:col10/1732481321789/Put/seqid=0 2024-11-24T20:48:42,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742264_1440 (size=12151) 2024-11-24T20:48:42,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T20:48:42,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:42,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:42,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:42,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:42,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:42,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:42,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T20:48:42,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:42,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:42,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:42,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:42,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:42,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:42,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T20:48:42,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:42,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481382475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:42,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:42,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:42,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:42,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:42,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481382485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:42,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:42,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:42,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:42,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T20:48:42,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:42,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:42,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:42,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:42,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:42,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:42,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/42d92306bf314ff3b6917c7f8255dea2 2024-11-24T20:48:42,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/e7137814d4134479bfac9626b43e8e0c is 50, key is test_row_0/C:col10/1732481321789/Put/seqid=0 2024-11-24T20:48:42,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742265_1441 (size=12151) 2024-11-24T20:48:42,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:42,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51214 deadline: 1732481382725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:42,730 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_0' in 94ec587900a8a4e84a03f1b687bbae49 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:48:42,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:42,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:42,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:42,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:42,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51228 deadline: 1732481382735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:42,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T20:48:42,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:42,743 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_0' in 94ec587900a8a4e84a03f1b687bbae49 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:48:42,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:42,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:42,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:42,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:42,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:42,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:42,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:42,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:42,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:42,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51226 deadline: 1732481382746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:42,753 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_2' in 94ec587900a8a4e84a03f1b687bbae49 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:48:42,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:42,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:42,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:42,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T20:48:42,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:42,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:42,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:42,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:42,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:42,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:42,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T20:48:42,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:42,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481382985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:42,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:42,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:42,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:43,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:43,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481382995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:43,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:43,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:43,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:43,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/e7137814d4134479bfac9626b43e8e0c 2024-11-24T20:48:43,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/1231c89e5d6f499a92190db3eea5cbcd as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/1231c89e5d6f499a92190db3eea5cbcd 2024-11-24T20:48:43,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/1231c89e5d6f499a92190db3eea5cbcd, entries=200, sequenceid=152, filesize=14.2 K 2024-11-24T20:48:43,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/42d92306bf314ff3b6917c7f8255dea2 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/42d92306bf314ff3b6917c7f8255dea2 2024-11-24T20:48:43,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/42d92306bf314ff3b6917c7f8255dea2, entries=150, sequenceid=152, filesize=11.9 K 2024-11-24T20:48:43,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/e7137814d4134479bfac9626b43e8e0c as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/e7137814d4134479bfac9626b43e8e0c 2024-11-24T20:48:43,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/e7137814d4134479bfac9626b43e8e0c, entries=150, sequenceid=152, filesize=11.9 K 2024-11-24T20:48:43,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 94ec587900a8a4e84a03f1b687bbae49 in 1249ms, sequenceid=152, compaction requested=true 2024-11-24T20:48:43,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:43,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:48:43,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:43,039 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:48:43,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:48:43,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:43,039 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:48:43,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:48:43,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:43,040 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:48:43,040 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55530 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:48:43,040 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/A is initiating minor compaction (all files) 2024-11-24T20:48:43,040 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/B is initiating minor compaction (all files) 2024-11-24T20:48:43,040 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/B in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:43,040 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/A in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:43,040 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/b10f6d1da31f497a98fd53463cce39d7, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/a39eb05eedcf42b5a6b82deddb07b700, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/fb185c86f8f54d6ba26c7207b7b1c21b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/42d92306bf314ff3b6917c7f8255dea2] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=47.3 K 2024-11-24T20:48:43,040 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/28f34beb11c248ffba737881342d4af1, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/87657b74b75e4f1c8cce8cd89d38ec94, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/ac65fa7f5bcd4df2a5223744718cf7d4, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/1231c89e5d6f499a92190db3eea5cbcd] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=54.2 K 2024-11-24T20:48:43,040 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting b10f6d1da31f497a98fd53463cce39d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732481319122 2024-11-24T20:48:43,040 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28f34beb11c248ffba737881342d4af1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732481319122 2024-11-24T20:48:43,040 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting a39eb05eedcf42b5a6b82deddb07b700, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732481320328 2024-11-24T20:48:43,040 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87657b74b75e4f1c8cce8cd89d38ec94, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732481320292 2024-11-24T20:48:43,041 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting fb185c86f8f54d6ba26c7207b7b1c21b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732481320417 2024-11-24T20:48:43,041 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac65fa7f5bcd4df2a5223744718cf7d4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732481320417 2024-11-24T20:48:43,041 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 42d92306bf314ff3b6917c7f8255dea2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732481321141 2024-11-24T20:48:43,041 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1231c89e5d6f499a92190db3eea5cbcd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732481321140 2024-11-24T20:48:43,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T20:48:43,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:43,049 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#B#compaction#371 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:43,049 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 94ec587900a8a4e84a03f1b687bbae49 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-24T20:48:43,049 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#A#compaction#372 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:43,049 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/af1254ac3a55496991387c79fe1800e8 is 50, key is test_row_0/B:col10/1732481321789/Put/seqid=0 2024-11-24T20:48:43,050 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/780a9c774a8341988fa4ed65a051f7c7 is 50, key is test_row_0/A:col10/1732481321789/Put/seqid=0 2024-11-24T20:48:43,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/a71bb620920a4cf9ae81aa62dfff465b is 50, key is test_row_0/A:col10/1732481321842/Put/seqid=0 2024-11-24T20:48:43,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742267_1443 (size=12493) 2024-11-24T20:48:43,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742266_1442 (size=12493) 2024-11-24T20:48:43,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742268_1444 (size=12151) 2024-11-24T20:48:43,066 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/a71bb620920a4cf9ae81aa62dfff465b 2024-11-24T20:48:43,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/06fe13e558934a538b85e5ae3776ed42 is 50, key is test_row_0/B:col10/1732481321842/Put/seqid=0 2024-11-24T20:48:43,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742269_1445 (size=12151) 2024-11-24T20:48:43,461 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/af1254ac3a55496991387c79fe1800e8 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/af1254ac3a55496991387c79fe1800e8 2024-11-24T20:48:43,475 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/06fe13e558934a538b85e5ae3776ed42 2024-11-24T20:48:43,563 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/B of 94ec587900a8a4e84a03f1b687bbae49 into af1254ac3a55496991387c79fe1800e8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:43,563 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:43,563 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/B, priority=12, startTime=1732481323039; duration=0sec 2024-11-24T20:48:43,563 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:43,563 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:B 2024-11-24T20:48:43,564 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:48:43,565 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:48:43,565 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/C is initiating minor compaction (all files) 2024-11-24T20:48:43,565 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/C in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:43,565 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/f2a0b69d54ca43959f0f0dc32d9f0186, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/7f6207dd546c4e12a2bf7ed757d0c1b9, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/dd832261c2864a52b981cca1d7fa9701, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/e7137814d4134479bfac9626b43e8e0c] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=47.3 K 2024-11-24T20:48:43,565 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/780a9c774a8341988fa4ed65a051f7c7 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/780a9c774a8341988fa4ed65a051f7c7 2024-11-24T20:48:43,566 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting f2a0b69d54ca43959f0f0dc32d9f0186, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732481319122 2024-11-24T20:48:43,566 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f6207dd546c4e12a2bf7ed757d0c1b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732481320328 2024-11-24T20:48:43,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/a66eda88dd484ca7be59112a563c8faf is 50, key is test_row_0/C:col10/1732481321842/Put/seqid=0 2024-11-24T20:48:43,568 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting dd832261c2864a52b981cca1d7fa9701, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732481320417 2024-11-24T20:48:43,568 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting e7137814d4134479bfac9626b43e8e0c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732481321141 2024-11-24T20:48:43,573 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/A of 94ec587900a8a4e84a03f1b687bbae49 into 780a9c774a8341988fa4ed65a051f7c7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:43,573 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:43,573 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/A, priority=12, startTime=1732481323039; duration=0sec 2024-11-24T20:48:43,573 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:43,573 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:A 2024-11-24T20:48:43,585 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#C#compaction#376 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:43,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742270_1446 (size=12151) 2024-11-24T20:48:43,585 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/54e88b2cbeea4852897a74e7e9f370ea is 50, key is test_row_0/C:col10/1732481321789/Put/seqid=0 2024-11-24T20:48:43,586 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/a66eda88dd484ca7be59112a563c8faf 2024-11-24T20:48:43,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742271_1447 (size=12493) 2024-11-24T20:48:43,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/a71bb620920a4cf9ae81aa62dfff465b as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/a71bb620920a4cf9ae81aa62dfff465b 2024-11-24T20:48:43,594 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/a71bb620920a4cf9ae81aa62dfff465b, entries=150, sequenceid=168, filesize=11.9 K 2024-11-24T20:48:43,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/06fe13e558934a538b85e5ae3776ed42 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/06fe13e558934a538b85e5ae3776ed42 2024-11-24T20:48:43,598 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/06fe13e558934a538b85e5ae3776ed42, entries=150, sequenceid=168, filesize=11.9 K 2024-11-24T20:48:43,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/a66eda88dd484ca7be59112a563c8faf as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/a66eda88dd484ca7be59112a563c8faf 2024-11-24T20:48:43,602 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/a66eda88dd484ca7be59112a563c8faf, entries=150, sequenceid=168, filesize=11.9 K 2024-11-24T20:48:43,602 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=0 B/0 for 94ec587900a8a4e84a03f1b687bbae49 in 553ms, sequenceid=168, compaction requested=false 2024-11-24T20:48:43,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:43,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:43,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-24T20:48:43,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-24T20:48:43,605 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-24T20:48:43,605 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7760 sec 2024-11-24T20:48:43,606 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.7810 sec 2024-11-24T20:48:43,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T20:48:43,975 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:48:43,975 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:48:43,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-24T20:48:43,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T20:48:43,977 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:48:43,978 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:48:43,978 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:48:44,000 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/54e88b2cbeea4852897a74e7e9f370ea as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/54e88b2cbeea4852897a74e7e9f370ea 2024-11-24T20:48:44,005 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/C of 94ec587900a8a4e84a03f1b687bbae49 into 54e88b2cbeea4852897a74e7e9f370ea(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:44,005 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:44,005 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/C, priority=12, startTime=1732481323039; duration=0sec 2024-11-24T20:48:44,006 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:44,006 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:C 2024-11-24T20:48:44,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:44,031 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:48:44,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94ec587900a8a4e84a03f1b687bbae49 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T20:48:44,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/9d99bd2223a146da8ce92b8b175e540b is 50, key is test_row_0/A:col10/1732481324008/Put/seqid=0 2024-11-24T20:48:44,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742272_1448 (size=16931) 2024-11-24T20:48:44,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T20:48:44,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-24T20:48:44,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:44,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:44,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:44,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:44,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:44,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:44,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:44,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481384136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:44,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:44,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:44,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481384140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:44,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:44,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:44,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:44,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:44,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481384255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:44,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:44,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:44,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:44,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:44,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481384255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:44,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:44,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:44,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:44,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-24T20:48:44,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:44,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:44,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:44,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:44,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T20:48:44,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-24T20:48:44,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:44,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:44,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:44,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:44,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:44,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:44,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/9d99bd2223a146da8ce92b8b175e540b 2024-11-24T20:48:44,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/63f19c2ae6e2457fa8bcad9a055f134f is 50, key is test_row_0/B:col10/1732481324008/Put/seqid=0 2024-11-24T20:48:44,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742273_1449 (size=12151) 2024-11-24T20:48:44,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/63f19c2ae6e2457fa8bcad9a055f134f 2024-11-24T20:48:44,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:44,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481384465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:44,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:44,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:44,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:44,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/71d8acbd8f704008ade84854ddaa42c1 is 50, key is test_row_0/C:col10/1732481324008/Put/seqid=0 2024-11-24T20:48:44,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742274_1450 (size=12151) 2024-11-24T20:48:44,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:44,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481384475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:44,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:44,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:44,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:44,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-24T20:48:44,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:44,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:44,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:44,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:44,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:44,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T20:48:44,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-24T20:48:44,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:44,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:44,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:44,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:44,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:44,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:44,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:44,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481384775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:44,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:44,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:44,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:44,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:44,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481384785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:44,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:44,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:44,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:44,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/71d8acbd8f704008ade84854ddaa42c1 2024-11-24T20:48:44,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/9d99bd2223a146da8ce92b8b175e540b as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/9d99bd2223a146da8ce92b8b175e540b 2024-11-24T20:48:44,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/9d99bd2223a146da8ce92b8b175e540b, entries=250, sequenceid=183, filesize=16.5 K 2024-11-24T20:48:44,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/63f19c2ae6e2457fa8bcad9a055f134f as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/63f19c2ae6e2457fa8bcad9a055f134f 2024-11-24T20:48:44,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/63f19c2ae6e2457fa8bcad9a055f134f, entries=150, sequenceid=183, filesize=11.9 K 2024-11-24T20:48:44,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/71d8acbd8f704008ade84854ddaa42c1 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/71d8acbd8f704008ade84854ddaa42c1 2024-11-24T20:48:44,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/71d8acbd8f704008ade84854ddaa42c1, entries=150, sequenceid=183, filesize=11.9 K 2024-11-24T20:48:44,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 94ec587900a8a4e84a03f1b687bbae49 in 863ms, sequenceid=183, compaction requested=true 2024-11-24T20:48:44,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:44,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:48:44,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:44,894 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:44,894 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:44,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:48:44,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:44,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:48:44,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:44,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-24T20:48:44,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:44,896 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 94ec587900a8a4e84a03f1b687bbae49 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-24T20:48:44,896 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41575 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:44,896 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:44,896 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/B is initiating minor compaction (all files) 2024-11-24T20:48:44,896 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/A is initiating minor compaction (all files) 2024-11-24T20:48:44,896 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/B in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:44,896 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/A in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:44,897 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/780a9c774a8341988fa4ed65a051f7c7, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/a71bb620920a4cf9ae81aa62dfff465b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/9d99bd2223a146da8ce92b8b175e540b] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=40.6 K 2024-11-24T20:48:44,897 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/af1254ac3a55496991387c79fe1800e8, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/06fe13e558934a538b85e5ae3776ed42, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/63f19c2ae6e2457fa8bcad9a055f134f] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=35.9 K 2024-11-24T20:48:44,897 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting af1254ac3a55496991387c79fe1800e8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732481321141 2024-11-24T20:48:44,897 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 780a9c774a8341988fa4ed65a051f7c7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732481321141 2024-11-24T20:48:44,897 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 06fe13e558934a538b85e5ae3776ed42, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732481321815 2024-11-24T20:48:44,897 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting a71bb620920a4cf9ae81aa62dfff465b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732481321815 2024-11-24T20:48:44,898 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 63f19c2ae6e2457fa8bcad9a055f134f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732481324008 2024-11-24T20:48:44,899 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d99bd2223a146da8ce92b8b175e540b, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732481324008 2024-11-24T20:48:44,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/51e907bc14e94cc2a870e393ae6a226e is 50, key is test_row_0/A:col10/1732481324139/Put/seqid=0 2024-11-24T20:48:44,913 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#B#compaction#381 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:44,914 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/3d2a7f59cb484c598fe6c48ee7a58a93 is 50, key is test_row_0/B:col10/1732481324008/Put/seqid=0 2024-11-24T20:48:44,926 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#A#compaction#382 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:44,927 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/dce8023b56cb45adaaf5fd5090ffee07 is 50, key is test_row_0/A:col10/1732481324008/Put/seqid=0 2024-11-24T20:48:44,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742275_1451 (size=12151) 2024-11-24T20:48:44,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742276_1452 (size=12595) 2024-11-24T20:48:44,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742277_1453 (size=12595) 2024-11-24T20:48:45,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T20:48:45,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:45,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:45,330 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/51e907bc14e94cc2a870e393ae6a226e 2024-11-24T20:48:45,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/dda5a6eb75c3458daaef6bed44327285 is 50, key is test_row_0/B:col10/1732481324139/Put/seqid=0 2024-11-24T20:48:45,338 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/3d2a7f59cb484c598fe6c48ee7a58a93 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/3d2a7f59cb484c598fe6c48ee7a58a93 2024-11-24T20:48:45,343 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/B of 94ec587900a8a4e84a03f1b687bbae49 into 3d2a7f59cb484c598fe6c48ee7a58a93(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:45,343 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:45,343 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/B, priority=13, startTime=1732481324894; duration=0sec 2024-11-24T20:48:45,343 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:45,343 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:B 2024-11-24T20:48:45,343 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:45,344 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:45,344 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/C is initiating minor compaction (all files) 2024-11-24T20:48:45,344 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/C in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:45,344 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/54e88b2cbeea4852897a74e7e9f370ea, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/a66eda88dd484ca7be59112a563c8faf, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/71d8acbd8f704008ade84854ddaa42c1] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=35.9 K 2024-11-24T20:48:45,345 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 54e88b2cbeea4852897a74e7e9f370ea, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732481321141 2024-11-24T20:48:45,345 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting a66eda88dd484ca7be59112a563c8faf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732481321815 2024-11-24T20:48:45,346 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 71d8acbd8f704008ade84854ddaa42c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732481324008 2024-11-24T20:48:45,349 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/dce8023b56cb45adaaf5fd5090ffee07 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/dce8023b56cb45adaaf5fd5090ffee07 2024-11-24T20:48:45,354 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/A of 94ec587900a8a4e84a03f1b687bbae49 into dce8023b56cb45adaaf5fd5090ffee07(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:45,355 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:45,355 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/A, priority=13, startTime=1732481324894; duration=0sec 2024-11-24T20:48:45,355 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:45,355 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:A 2024-11-24T20:48:45,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:45,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481385346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:45,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:45,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:45,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:45,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:45,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481385355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:45,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:45,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:45,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:45,378 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#C#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:45,379 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/d4930a21a9aa4209bda7adaed4e81021 is 50, key is test_row_0/C:col10/1732481324008/Put/seqid=0 2024-11-24T20:48:45,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742278_1454 (size=12151) 2024-11-24T20:48:45,407 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/dda5a6eb75c3458daaef6bed44327285 2024-11-24T20:48:45,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/4d3b013925c5485e810668c8f7b348b5 is 50, key is test_row_0/C:col10/1732481324139/Put/seqid=0 2024-11-24T20:48:45,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742279_1455 (size=12595) 2024-11-24T20:48:45,437 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/d4930a21a9aa4209bda7adaed4e81021 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/d4930a21a9aa4209bda7adaed4e81021 2024-11-24T20:48:45,443 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/C of 94ec587900a8a4e84a03f1b687bbae49 into d4930a21a9aa4209bda7adaed4e81021(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:45,443 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:45,443 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/C, priority=13, startTime=1732481324894; duration=0sec 2024-11-24T20:48:45,443 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:45,443 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:C 2024-11-24T20:48:45,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742280_1456 (size=12151) 2024-11-24T20:48:45,461 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/4d3b013925c5485e810668c8f7b348b5 2024-11-24T20:48:45,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/51e907bc14e94cc2a870e393ae6a226e as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/51e907bc14e94cc2a870e393ae6a226e 2024-11-24T20:48:45,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481385466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:45,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:45,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:45,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:45,478 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/51e907bc14e94cc2a870e393ae6a226e, entries=150, sequenceid=207, filesize=11.9 K 2024-11-24T20:48:45,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481385475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:45,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:45,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:45,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:45,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/dda5a6eb75c3458daaef6bed44327285 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/dda5a6eb75c3458daaef6bed44327285 2024-11-24T20:48:45,498 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/dda5a6eb75c3458daaef6bed44327285, entries=150, sequenceid=207, filesize=11.9 K 2024-11-24T20:48:45,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/4d3b013925c5485e810668c8f7b348b5 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/4d3b013925c5485e810668c8f7b348b5 2024-11-24T20:48:45,529 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/4d3b013925c5485e810668c8f7b348b5, entries=150, sequenceid=207, filesize=11.9 K 2024-11-24T20:48:45,530 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 94ec587900a8a4e84a03f1b687bbae49 in 634ms, sequenceid=207, compaction requested=false 2024-11-24T20:48:45,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:45,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:45,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-24T20:48:45,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-24T20:48:45,533 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-24T20:48:45,533 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5530 sec 2024-11-24T20:48:45,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.5600 sec 2024-11-24T20:48:45,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:45,688 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:48:45,688 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94ec587900a8a4e84a03f1b687bbae49 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-24T20:48:45,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/f7686677454e4a198d43aff48327e783 is 50, key is test_row_0/A:col10/1732481325343/Put/seqid=0 2024-11-24T20:48:45,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742281_1457 (size=14541) 2024-11-24T20:48:45,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/f7686677454e4a198d43aff48327e783 2024-11-24T20:48:45,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/6f88b495acaa4ce9aec614121c45f275 is 50, key is test_row_0/B:col10/1732481325343/Put/seqid=0 2024-11-24T20:48:45,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742282_1458 (size=12151) 2024-11-24T20:48:45,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/6f88b495acaa4ce9aec614121c45f275 2024-11-24T20:48:45,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/eef4834e70db40fba36b949362e70ba9 is 50, key is test_row_0/C:col10/1732481325343/Put/seqid=0 2024-11-24T20:48:45,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742283_1459 (size=12151) 2024-11-24T20:48:45,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:45,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481385824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:45,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:45,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:45,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:45,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:45,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481385826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:45,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:45,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:45,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:45,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:45,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481385935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:45,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:45,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:45,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:45,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:45,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481385945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:45,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T20:48:46,136 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:48:46,136 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:48:46,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-24T20:48:46,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T20:48:46,142 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:48:46,143 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:48:46,143 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:48:46,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/eef4834e70db40fba36b949362e70ba9 2024-11-24T20:48:46,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/f7686677454e4a198d43aff48327e783 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/f7686677454e4a198d43aff48327e783 2024-11-24T20:48:46,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/f7686677454e4a198d43aff48327e783, entries=200, sequenceid=223, filesize=14.2 K 2024-11-24T20:48:46,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/6f88b495acaa4ce9aec614121c45f275 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/6f88b495acaa4ce9aec614121c45f275 2024-11-24T20:48:46,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:46,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:46,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481386155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:46,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481386155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:46,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:46,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:46,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:46,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:46,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:46,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:46,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/6f88b495acaa4ce9aec614121c45f275, entries=150, sequenceid=223, filesize=11.9 K 2024-11-24T20:48:46,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/eef4834e70db40fba36b949362e70ba9 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/eef4834e70db40fba36b949362e70ba9 2024-11-24T20:48:46,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/eef4834e70db40fba36b949362e70ba9, entries=150, sequenceid=223, filesize=11.9 K 2024-11-24T20:48:46,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 94ec587900a8a4e84a03f1b687bbae49 in 484ms, sequenceid=223, compaction requested=true 2024-11-24T20:48:46,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:46,172 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:46,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:48:46,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:46,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:48:46,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:46,173 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:46,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:48:46,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:46,175 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:46,175 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:46,175 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/A is initiating minor compaction (all files) 2024-11-24T20:48:46,175 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/B is initiating minor compaction (all files) 2024-11-24T20:48:46,175 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/B in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:46,175 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/3d2a7f59cb484c598fe6c48ee7a58a93, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/dda5a6eb75c3458daaef6bed44327285, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/6f88b495acaa4ce9aec614121c45f275] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=36.0 K 2024-11-24T20:48:46,175 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/A in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:46,175 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/dce8023b56cb45adaaf5fd5090ffee07, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/51e907bc14e94cc2a870e393ae6a226e, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/f7686677454e4a198d43aff48327e783] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=38.4 K 2024-11-24T20:48:46,176 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d2a7f59cb484c598fe6c48ee7a58a93, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732481324008 2024-11-24T20:48:46,176 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting dce8023b56cb45adaaf5fd5090ffee07, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732481324008 2024-11-24T20:48:46,176 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting dda5a6eb75c3458daaef6bed44327285, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732481324125 2024-11-24T20:48:46,177 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f88b495acaa4ce9aec614121c45f275, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732481325343 2024-11-24T20:48:46,178 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51e907bc14e94cc2a870e393ae6a226e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732481324125 2024-11-24T20:48:46,178 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7686677454e4a198d43aff48327e783, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732481325335 2024-11-24T20:48:46,202 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#B#compaction#389 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:46,203 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/1cd4a4de3a844d1d936b80d464142a57 is 50, key is test_row_0/B:col10/1732481325343/Put/seqid=0 2024-11-24T20:48:46,213 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#A#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:46,214 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/9f37a066ca78440e924162f6b7026ecd is 50, key is test_row_0/A:col10/1732481325343/Put/seqid=0 2024-11-24T20:48:46,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T20:48:46,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742284_1460 (size=12697) 2024-11-24T20:48:46,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742285_1461 (size=12697) 2024-11-24T20:48:46,288 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/9f37a066ca78440e924162f6b7026ecd as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/9f37a066ca78440e924162f6b7026ecd 2024-11-24T20:48:46,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-24T20:48:46,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:46,298 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 94ec587900a8a4e84a03f1b687bbae49 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-24T20:48:46,300 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/A of 94ec587900a8a4e84a03f1b687bbae49 into 9f37a066ca78440e924162f6b7026ecd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:46,300 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:46,300 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/A, priority=13, startTime=1732481326172; duration=0sec 2024-11-24T20:48:46,300 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:46,300 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:A 2024-11-24T20:48:46,300 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:46,303 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:46,303 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/C is initiating minor compaction (all files) 2024-11-24T20:48:46,303 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/C in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:46,303 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/d4930a21a9aa4209bda7adaed4e81021, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/4d3b013925c5485e810668c8f7b348b5, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/eef4834e70db40fba36b949362e70ba9] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=36.0 K 2024-11-24T20:48:46,304 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4930a21a9aa4209bda7adaed4e81021, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732481324008 2024-11-24T20:48:46,304 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d3b013925c5485e810668c8f7b348b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732481324125 2024-11-24T20:48:46,305 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting eef4834e70db40fba36b949362e70ba9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732481325343 2024-11-24T20:48:46,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/0b2b93ee8d9841f681247ab8451f2901 is 50, key is test_row_0/A:col10/1732481325823/Put/seqid=0 2024-11-24T20:48:46,326 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#C#compaction#392 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:46,327 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/0cac906f64924b4abb15596ad45aff8d is 50, key is test_row_0/C:col10/1732481325343/Put/seqid=0 2024-11-24T20:48:46,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742286_1462 (size=12151) 2024-11-24T20:48:46,362 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/0b2b93ee8d9841f681247ab8451f2901 2024-11-24T20:48:46,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742287_1463 (size=12697) 2024-11-24T20:48:46,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/00abad3b91464e5ab93784e358e12fdc is 50, key is test_row_0/B:col10/1732481325823/Put/seqid=0 2024-11-24T20:48:46,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742288_1464 (size=12151) 2024-11-24T20:48:46,415 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/00abad3b91464e5ab93784e358e12fdc 2024-11-24T20:48:46,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/33334a29518142f5b66105aa510f682b is 50, key is test_row_0/C:col10/1732481325823/Put/seqid=0 2024-11-24T20:48:46,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742289_1465 (size=12151) 2024-11-24T20:48:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T20:48:46,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:46,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:46,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:46,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481386545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:46,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:46,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:46,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:46,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:46,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481386547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:46,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:46,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:46,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:46,655 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/1cd4a4de3a844d1d936b80d464142a57 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/1cd4a4de3a844d1d936b80d464142a57 2024-11-24T20:48:46,659 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/B of 94ec587900a8a4e84a03f1b687bbae49 into 1cd4a4de3a844d1d936b80d464142a57(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:46,659 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:46,659 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/B, priority=13, startTime=1732481326173; duration=0sec 2024-11-24T20:48:46,660 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:46,660 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:B 2024-11-24T20:48:46,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:46,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481386666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:46,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:46,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:46,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:46,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:46,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481386675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:46,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:46,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:46,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:46,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:46,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51214 deadline: 1732481386755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:46,764 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_0' in 94ec587900a8a4e84a03f1b687bbae49 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:48:46,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:46,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:46,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:46,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T20:48:46,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:46,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51226 deadline: 1732481386765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:46,778 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_2' in 94ec587900a8a4e84a03f1b687bbae49 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:48:46,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:46,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:46,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:46,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:46,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51228 deadline: 1732481386775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:46,780 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_0' in 94ec587900a8a4e84a03f1b687bbae49 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:48:46,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:46,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:46,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:46,793 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/0cac906f64924b4abb15596ad45aff8d as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/0cac906f64924b4abb15596ad45aff8d 2024-11-24T20:48:46,797 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/C of 94ec587900a8a4e84a03f1b687bbae49 into 0cac906f64924b4abb15596ad45aff8d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:46,797 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:46,797 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/C, priority=13, startTime=1732481326173; duration=0sec 2024-11-24T20:48:46,797 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:46,797 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:C 2024-11-24T20:48:46,830 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/33334a29518142f5b66105aa510f682b 2024-11-24T20:48:46,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/0b2b93ee8d9841f681247ab8451f2901 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/0b2b93ee8d9841f681247ab8451f2901 2024-11-24T20:48:46,838 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/0b2b93ee8d9841f681247ab8451f2901, entries=150, sequenceid=247, filesize=11.9 K 2024-11-24T20:48:46,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-24T20:48:46,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/00abad3b91464e5ab93784e358e12fdc as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/00abad3b91464e5ab93784e358e12fdc 2024-11-24T20:48:46,844 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/00abad3b91464e5ab93784e358e12fdc, entries=150, sequenceid=247, filesize=11.9 K 2024-11-24T20:48:46,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/33334a29518142f5b66105aa510f682b as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/33334a29518142f5b66105aa510f682b 2024-11-24T20:48:46,851 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/33334a29518142f5b66105aa510f682b, entries=150, sequenceid=247, filesize=11.9 K 2024-11-24T20:48:46,852 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 94ec587900a8a4e84a03f1b687bbae49 in 554ms, sequenceid=247, compaction requested=false 2024-11-24T20:48:46,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:46,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:46,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-24T20:48:46,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-24T20:48:46,856 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-24T20:48:46,856 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 711 msec 2024-11-24T20:48:46,860 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 722 msec 2024-11-24T20:48:46,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:46,891 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:48:46,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94ec587900a8a4e84a03f1b687bbae49 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-24T20:48:46,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/b018486889cf458eb6292e2c1489d42a is 50, key is test_row_0/A:col10/1732481326545/Put/seqid=0 2024-11-24T20:48:46,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742290_1466 (size=14741) 2024-11-24T20:48:47,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:47,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481387005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:47,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:47,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:47,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:47,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:47,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481387013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:47,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:47,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:47,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:47,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:47,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481387125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:47,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:47,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:47,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:47,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481387125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:47,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:47,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:47,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:47,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:47,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T20:48:47,275 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:48:47,276 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:48:47,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-24T20:48:47,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T20:48:47,277 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:48:47,278 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:48:47,278 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:48:47,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/b018486889cf458eb6292e2c1489d42a 2024-11-24T20:48:47,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/827cf1638dfc4ca7ab80c9c7bd9e8f00 is 50, key is test_row_0/B:col10/1732481326545/Put/seqid=0 2024-11-24T20:48:47,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742291_1467 (size=12301) 2024-11-24T20:48:47,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:47,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481387345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:47,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:47,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:47,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:47,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:47,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481387345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:47,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:47,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:47,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:47,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T20:48:47,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-24T20:48:47,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:47,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:47,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:47,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:47,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:47,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-24T20:48:47,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:47,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:47,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:47,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:47,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T20:48:47,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481387655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:47,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481387655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:47,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:47,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:47,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:47,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:47,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:47,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:47,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/827cf1638dfc4ca7ab80c9c7bd9e8f00 2024-11-24T20:48:47,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/019def6faf8445c6b684ae7a75e7e6bc is 50, key is test_row_0/C:col10/1732481326545/Put/seqid=0 2024-11-24T20:48:47,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742292_1468 (size=12301) 2024-11-24T20:48:47,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-24T20:48:47,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:47,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:47,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:47,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:47,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:47,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-24T20:48:47,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:47,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:47,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:47,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:47,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:47,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:47,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T20:48:48,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-24T20:48:48,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:48,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:48,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:48,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:48,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:48,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:48,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/019def6faf8445c6b684ae7a75e7e6bc 2024-11-24T20:48:48,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/b018486889cf458eb6292e2c1489d42a as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/b018486889cf458eb6292e2c1489d42a 2024-11-24T20:48:48,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/b018486889cf458eb6292e2c1489d42a, entries=200, sequenceid=263, filesize=14.4 K 2024-11-24T20:48:48,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/827cf1638dfc4ca7ab80c9c7bd9e8f00 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/827cf1638dfc4ca7ab80c9c7bd9e8f00 2024-11-24T20:48:48,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/827cf1638dfc4ca7ab80c9c7bd9e8f00, entries=150, sequenceid=263, filesize=12.0 K 2024-11-24T20:48:48,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/019def6faf8445c6b684ae7a75e7e6bc as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/019def6faf8445c6b684ae7a75e7e6bc 2024-11-24T20:48:48,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/019def6faf8445c6b684ae7a75e7e6bc, entries=150, sequenceid=263, filesize=12.0 K 2024-11-24T20:48:48,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 94ec587900a8a4e84a03f1b687bbae49 in 1269ms, sequenceid=263, compaction requested=true 2024-11-24T20:48:48,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:48,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:48:48,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:48,160 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:48,160 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:48,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:48:48,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:48,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:48:48,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:48,169 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39589 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:48,169 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/A is initiating minor compaction (all files) 2024-11-24T20:48:48,169 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/A in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:48,169 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/9f37a066ca78440e924162f6b7026ecd, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/0b2b93ee8d9841f681247ab8451f2901, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/b018486889cf458eb6292e2c1489d42a] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=38.7 K 2024-11-24T20:48:48,170 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:48,170 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/B is initiating minor compaction (all files) 2024-11-24T20:48:48,170 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f37a066ca78440e924162f6b7026ecd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732481325343 2024-11-24T20:48:48,170 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/B in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:48,170 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/1cd4a4de3a844d1d936b80d464142a57, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/00abad3b91464e5ab93784e358e12fdc, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/827cf1638dfc4ca7ab80c9c7bd9e8f00] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=36.3 K 2024-11-24T20:48:48,170 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b2b93ee8d9841f681247ab8451f2901, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732481325809 2024-11-24T20:48:48,170 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cd4a4de3a844d1d936b80d464142a57, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732481325343 2024-11-24T20:48:48,170 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting b018486889cf458eb6292e2c1489d42a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732481326530 2024-11-24T20:48:48,170 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 00abad3b91464e5ab93784e358e12fdc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732481325809 2024-11-24T20:48:48,171 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 827cf1638dfc4ca7ab80c9c7bd9e8f00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732481326543 2024-11-24T20:48:48,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:48,179 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:48:48,179 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94ec587900a8a4e84a03f1b687bbae49 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-24T20:48:48,186 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#A#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:48,187 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/ff5ccb90be9b44afbae1a7ba7e34cbb4 is 50, key is test_row_0/A:col10/1732481326545/Put/seqid=0 2024-11-24T20:48:48,197 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#B#compaction#399 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:48,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-24T20:48:48,197 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/3366ab9c30094e5faa6c02038f30d879 is 50, key is test_row_0/B:col10/1732481326545/Put/seqid=0 2024-11-24T20:48:48,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:48,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:48,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:48,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:48,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:48,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/fbc20bfcf68445369e77da12c8ff20bf is 50, key is test_row_0/A:col10/1732481326987/Put/seqid=0 2024-11-24T20:48:48,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742293_1469 (size=12949) 2024-11-24T20:48:48,228 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/ff5ccb90be9b44afbae1a7ba7e34cbb4 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/ff5ccb90be9b44afbae1a7ba7e34cbb4 2024-11-24T20:48:48,234 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/A of 94ec587900a8a4e84a03f1b687bbae49 into ff5ccb90be9b44afbae1a7ba7e34cbb4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:48,234 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:48,234 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/A, priority=13, startTime=1732481328160; duration=0sec 2024-11-24T20:48:48,234 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:48,234 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:A 2024-11-24T20:48:48,234 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:48,235 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:48,235 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/C is initiating minor compaction (all files) 2024-11-24T20:48:48,235 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/C in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:48,236 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/0cac906f64924b4abb15596ad45aff8d, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/33334a29518142f5b66105aa510f682b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/019def6faf8445c6b684ae7a75e7e6bc] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=36.3 K 2024-11-24T20:48:48,236 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cac906f64924b4abb15596ad45aff8d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732481325343 2024-11-24T20:48:48,236 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33334a29518142f5b66105aa510f682b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732481325809 2024-11-24T20:48:48,236 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 019def6faf8445c6b684ae7a75e7e6bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732481326543 2024-11-24T20:48:48,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742294_1470 (size=12949) 2024-11-24T20:48:48,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:48,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:48,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481388240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:48,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481388241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:48,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:48,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:48,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:48,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:48,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:48,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:48,254 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/3366ab9c30094e5faa6c02038f30d879 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/3366ab9c30094e5faa6c02038f30d879 2024-11-24T20:48:48,254 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#C#compaction#401 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:48,254 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/615be6ed8a484972aefa50975ea9d1c0 is 50, key is test_row_0/C:col10/1732481326545/Put/seqid=0 2024-11-24T20:48:48,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742295_1471 (size=14741) 2024-11-24T20:48:48,258 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/B of 94ec587900a8a4e84a03f1b687bbae49 into 3366ab9c30094e5faa6c02038f30d879(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:48,258 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:48,258 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/B, priority=13, startTime=1732481328160; duration=0sec 2024-11-24T20:48:48,259 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:48,259 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:B 2024-11-24T20:48:48,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/fbc20bfcf68445369e77da12c8ff20bf 2024-11-24T20:48:48,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742296_1472 (size=12949) 2024-11-24T20:48:48,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/412b3dbefbf04f0395eeeef67a060858 is 50, key is test_row_0/B:col10/1732481326987/Put/seqid=0 2024-11-24T20:48:48,271 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/615be6ed8a484972aefa50975ea9d1c0 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/615be6ed8a484972aefa50975ea9d1c0 2024-11-24T20:48:48,274 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/C of 94ec587900a8a4e84a03f1b687bbae49 into 615be6ed8a484972aefa50975ea9d1c0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:48,274 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:48,274 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/C, priority=13, startTime=1732481328161; duration=0sec 2024-11-24T20:48:48,274 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:48,274 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:C 2024-11-24T20:48:48,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742297_1473 (size=12301) 2024-11-24T20:48:48,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-24T20:48:48,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:48,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:48,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:48,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:48,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:48,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:48,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:48,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481388355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:48,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:48,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:48,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:48,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:48,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481388356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:48,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:48,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:48,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:48,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T20:48:48,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-24T20:48:48,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:48,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:48,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:48,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:48,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:48,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:48,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481388565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:48,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:48,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:48,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:48,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:48,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481388575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:48,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:48,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:48,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:48,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/412b3dbefbf04f0395eeeef67a060858 2024-11-24T20:48:48,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-24T20:48:48,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:48,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:48,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:48,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:48,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:48,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:48,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/9e48298c8e204f86a6760ff8839627ec is 50, key is test_row_0/C:col10/1732481326987/Put/seqid=0 2024-11-24T20:48:48,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742298_1474 (size=12301) 2024-11-24T20:48:48,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/9e48298c8e204f86a6760ff8839627ec 2024-11-24T20:48:48,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/fbc20bfcf68445369e77da12c8ff20bf as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/fbc20bfcf68445369e77da12c8ff20bf 2024-11-24T20:48:48,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/fbc20bfcf68445369e77da12c8ff20bf, entries=200, sequenceid=286, filesize=14.4 K 2024-11-24T20:48:48,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/412b3dbefbf04f0395eeeef67a060858 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/412b3dbefbf04f0395eeeef67a060858 2024-11-24T20:48:48,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/412b3dbefbf04f0395eeeef67a060858, entries=150, sequenceid=286, filesize=12.0 K 2024-11-24T20:48:48,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/9e48298c8e204f86a6760ff8839627ec as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/9e48298c8e204f86a6760ff8839627ec 2024-11-24T20:48:48,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/9e48298c8e204f86a6760ff8839627ec, entries=150, sequenceid=286, filesize=12.0 K 2024-11-24T20:48:48,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 94ec587900a8a4e84a03f1b687bbae49 in 544ms, sequenceid=286, compaction requested=false 2024-11-24T20:48:48,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:48,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-24T20:48:48,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:48,838 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 94ec587900a8a4e84a03f1b687bbae49 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-24T20:48:48,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/0d1f013ee541455d8413e52b86992506 is 50, key is test_row_0/A:col10/1732481328195/Put/seqid=0 2024-11-24T20:48:48,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742299_1475 (size=12301) 2024-11-24T20:48:48,844 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/0d1f013ee541455d8413e52b86992506 2024-11-24T20:48:48,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/6547c5329f0749539cda6c2e1d053b4a is 50, key is test_row_0/B:col10/1732481328195/Put/seqid=0 2024-11-24T20:48:48,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742300_1476 (size=12301) 2024-11-24T20:48:48,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:48,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:48,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:48,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481388935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:48,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:48,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:48,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:48,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:48,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481388939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:48,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:48,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:48,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:49,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:49,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481389045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:49,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:49,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:49,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:49,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481389055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:49,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:49,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:49,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:49,257 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/6547c5329f0749539cda6c2e1d053b4a 2024-11-24T20:48:49,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/271206f4a0b24da892bd6fd7365a3c21 is 50, key is test_row_0/C:col10/1732481328195/Put/seqid=0 2024-11-24T20:48:49,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:49,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481389265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:49,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:49,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:49,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:49,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:49,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481389275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:49,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:49,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:49,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:49,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742301_1477 (size=12301) 2024-11-24T20:48:49,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T20:48:49,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:49,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481389585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:49,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:49,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:49,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481389585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:49,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:49,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:49,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:49,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:49,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:49,686 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/271206f4a0b24da892bd6fd7365a3c21 2024-11-24T20:48:49,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/0d1f013ee541455d8413e52b86992506 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/0d1f013ee541455d8413e52b86992506 2024-11-24T20:48:49,694 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/0d1f013ee541455d8413e52b86992506, entries=150, sequenceid=302, filesize=12.0 K 2024-11-24T20:48:49,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/6547c5329f0749539cda6c2e1d053b4a as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/6547c5329f0749539cda6c2e1d053b4a 2024-11-24T20:48:49,699 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/6547c5329f0749539cda6c2e1d053b4a, entries=150, sequenceid=302, filesize=12.0 K 2024-11-24T20:48:49,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/271206f4a0b24da892bd6fd7365a3c21 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/271206f4a0b24da892bd6fd7365a3c21 2024-11-24T20:48:49,704 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/271206f4a0b24da892bd6fd7365a3c21, entries=150, sequenceid=302, filesize=12.0 K 2024-11-24T20:48:49,707 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 94ec587900a8a4e84a03f1b687bbae49 in 869ms, sequenceid=302, compaction requested=true 2024-11-24T20:48:49,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:49,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:49,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-24T20:48:49,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-24T20:48:49,711 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-24T20:48:49,711 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4310 sec 2024-11-24T20:48:49,713 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.4350 sec 2024-11-24T20:48:50,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:50,100 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:48:50,100 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94ec587900a8a4e84a03f1b687bbae49 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-24T20:48:50,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/0bcee7dd7397446ca90d9da3da78c5a5 is 50, key is test_row_0/A:col10/1732481330099/Put/seqid=0 2024-11-24T20:48:50,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742302_1478 (size=14741) 2024-11-24T20:48:50,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:50,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481390129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:50,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:50,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:50,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:50,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:50,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481390138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:50,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:50,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:50,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:50,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:50,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481390245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:50,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:50,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:50,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:50,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:50,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481390255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:50,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:50,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:50,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:50,444 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T20:48:50,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:50,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481390455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:50,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:50,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:50,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:50,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:50,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481390475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:50,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:50,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:50,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:50,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/0bcee7dd7397446ca90d9da3da78c5a5 2024-11-24T20:48:50,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/933b662931bd4faf9aeb5dac6ed5046f is 50, key is test_row_0/B:col10/1732481330099/Put/seqid=0 2024-11-24T20:48:50,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742303_1479 (size=12301) 2024-11-24T20:48:50,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/933b662931bd4faf9aeb5dac6ed5046f 2024-11-24T20:48:50,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/e061b23fae3844f484c31e3ac8b6bbdd is 50, key is test_row_0/C:col10/1732481330099/Put/seqid=0 2024-11-24T20:48:50,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742304_1480 (size=12301) 2024-11-24T20:48:50,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481390765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:50,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:50,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:50,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:50,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481390785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:50,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:50,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:50,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:50,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/e061b23fae3844f484c31e3ac8b6bbdd 2024-11-24T20:48:50,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/0bcee7dd7397446ca90d9da3da78c5a5 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/0bcee7dd7397446ca90d9da3da78c5a5 2024-11-24T20:48:50,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/0bcee7dd7397446ca90d9da3da78c5a5, entries=200, sequenceid=326, filesize=14.4 K 2024-11-24T20:48:50,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/933b662931bd4faf9aeb5dac6ed5046f as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/933b662931bd4faf9aeb5dac6ed5046f 2024-11-24T20:48:50,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/933b662931bd4faf9aeb5dac6ed5046f, entries=150, sequenceid=326, filesize=12.0 K 2024-11-24T20:48:50,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/e061b23fae3844f484c31e3ac8b6bbdd as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/e061b23fae3844f484c31e3ac8b6bbdd 2024-11-24T20:48:50,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/e061b23fae3844f484c31e3ac8b6bbdd, entries=150, sequenceid=326, filesize=12.0 K 2024-11-24T20:48:50,952 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 94ec587900a8a4e84a03f1b687bbae49 in 852ms, sequenceid=326, compaction requested=true 2024-11-24T20:48:50,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:50,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:48:50,952 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:48:50,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:50,952 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:48:50,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:48:50,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:50,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:48:50,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:50,953 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:48:50,953 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/B is initiating minor compaction (all files) 2024-11-24T20:48:50,953 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/B in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:50,953 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/3366ab9c30094e5faa6c02038f30d879, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/412b3dbefbf04f0395eeeef67a060858, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/6547c5329f0749539cda6c2e1d053b4a, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/933b662931bd4faf9aeb5dac6ed5046f] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=48.7 K 2024-11-24T20:48:50,954 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54732 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:48:50,954 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/A is initiating minor compaction (all files) 2024-11-24T20:48:50,954 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/A in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:50,954 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 3366ab9c30094e5faa6c02038f30d879, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732481326543 2024-11-24T20:48:50,954 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/ff5ccb90be9b44afbae1a7ba7e34cbb4, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/fbc20bfcf68445369e77da12c8ff20bf, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/0d1f013ee541455d8413e52b86992506, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/0bcee7dd7397446ca90d9da3da78c5a5] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=53.4 K 2024-11-24T20:48:50,954 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 412b3dbefbf04f0395eeeef67a060858, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732481326987 2024-11-24T20:48:50,954 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff5ccb90be9b44afbae1a7ba7e34cbb4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732481326543 2024-11-24T20:48:50,954 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 6547c5329f0749539cda6c2e1d053b4a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732481328195 2024-11-24T20:48:50,954 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbc20bfcf68445369e77da12c8ff20bf, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732481326935 2024-11-24T20:48:50,954 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 933b662931bd4faf9aeb5dac6ed5046f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732481328925 2024-11-24T20:48:50,955 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d1f013ee541455d8413e52b86992506, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732481328195 2024-11-24T20:48:50,955 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bcee7dd7397446ca90d9da3da78c5a5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732481328925 2024-11-24T20:48:50,961 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#B#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:50,962 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/9d7b1d1f9ba5435695440b4e722082a5 is 50, key is test_row_0/B:col10/1732481330099/Put/seqid=0 2024-11-24T20:48:50,962 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#A#compaction#411 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:50,963 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/b3de9083db63404ea043fab92f917b23 is 50, key is test_row_0/A:col10/1732481330099/Put/seqid=0 2024-11-24T20:48:50,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742305_1481 (size=13085) 2024-11-24T20:48:50,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742306_1482 (size=13085) 2024-11-24T20:48:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:51,286 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:48:51,286 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94ec587900a8a4e84a03f1b687bbae49 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-24T20:48:51,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/2e1b3474d572469c880798d6b26c62ed is 50, key is test_row_0/A:col10/1732481330124/Put/seqid=0 2024-11-24T20:48:51,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742307_1483 (size=14741) 2024-11-24T20:48:51,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/2e1b3474d572469c880798d6b26c62ed 2024-11-24T20:48:51,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/92f01fa196394e9f8ea91d3ce8f61497 is 50, key is test_row_0/B:col10/1732481330124/Put/seqid=0 2024-11-24T20:48:51,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742308_1484 (size=12301) 2024-11-24T20:48:51,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/92f01fa196394e9f8ea91d3ce8f61497 2024-11-24T20:48:51,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/9de4aa80d731420c8faf78f40123ce07 is 50, key is test_row_0/C:col10/1732481330124/Put/seqid=0 2024-11-24T20:48:51,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742309_1485 (size=12301) 2024-11-24T20:48:51,369 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/9d7b1d1f9ba5435695440b4e722082a5 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/9d7b1d1f9ba5435695440b4e722082a5 2024-11-24T20:48:51,373 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/b3de9083db63404ea043fab92f917b23 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/b3de9083db63404ea043fab92f917b23 2024-11-24T20:48:51,374 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/B of 94ec587900a8a4e84a03f1b687bbae49 into 9d7b1d1f9ba5435695440b4e722082a5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:51,374 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:51,374 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/B, priority=12, startTime=1732481330952; duration=0sec 2024-11-24T20:48:51,374 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:51,374 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:B 2024-11-24T20:48:51,374 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:48:51,375 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:48:51,375 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/C is initiating minor compaction (all files) 2024-11-24T20:48:51,375 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/C in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:51,375 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/615be6ed8a484972aefa50975ea9d1c0, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/9e48298c8e204f86a6760ff8839627ec, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/271206f4a0b24da892bd6fd7365a3c21, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/e061b23fae3844f484c31e3ac8b6bbdd] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=48.7 K 2024-11-24T20:48:51,376 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 615be6ed8a484972aefa50975ea9d1c0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732481326543 2024-11-24T20:48:51,376 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e48298c8e204f86a6760ff8839627ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732481326987 2024-11-24T20:48:51,376 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 271206f4a0b24da892bd6fd7365a3c21, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732481328195 2024-11-24T20:48:51,376 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting e061b23fae3844f484c31e3ac8b6bbdd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732481328925 2024-11-24T20:48:51,376 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/A of 94ec587900a8a4e84a03f1b687bbae49 into b3de9083db63404ea043fab92f917b23(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:51,376 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:51,376 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/A, priority=12, startTime=1732481330952; duration=0sec 2024-11-24T20:48:51,376 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:51,376 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:A 2024-11-24T20:48:51,390 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#C#compaction#415 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:51,390 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/5f218561758541c6b6ce0d598aa90d60 is 50, key is test_row_0/C:col10/1732481330099/Put/seqid=0 2024-11-24T20:48:51,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742310_1486 (size=13085) 2024-11-24T20:48:51,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:51,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481391406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481391406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:51,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:51,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:51,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:51,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:51,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:51,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T20:48:51,445 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:48:51,445 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:48:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-24T20:48:51,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T20:48:51,447 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:48:51,448 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:48:51,448 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:48:51,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481391524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:51,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:51,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:51,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:51,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481391525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:51,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:51,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:51,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T20:48:51,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-24T20:48:51,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:51,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:51,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:51,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:51,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:51,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/9de4aa80d731420c8faf78f40123ce07 2024-11-24T20:48:51,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/2e1b3474d572469c880798d6b26c62ed as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/2e1b3474d572469c880798d6b26c62ed 2024-11-24T20:48:51,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/2e1b3474d572469c880798d6b26c62ed, entries=200, sequenceid=339, filesize=14.4 K 2024-11-24T20:48:51,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/92f01fa196394e9f8ea91d3ce8f61497 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/92f01fa196394e9f8ea91d3ce8f61497 2024-11-24T20:48:51,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/92f01fa196394e9f8ea91d3ce8f61497, entries=150, sequenceid=339, filesize=12.0 K 2024-11-24T20:48:51,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/9de4aa80d731420c8faf78f40123ce07 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/9de4aa80d731420c8faf78f40123ce07 2024-11-24T20:48:51,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/9de4aa80d731420c8faf78f40123ce07, entries=150, sequenceid=339, filesize=12.0 K 2024-11-24T20:48:51,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 94ec587900a8a4e84a03f1b687bbae49 in 449ms, sequenceid=339, compaction requested=false 2024-11-24T20:48:51,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:51,744 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:48:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:51,744 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94ec587900a8a4e84a03f1b687bbae49 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-24T20:48:51,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/34bba7c371c8491381603161b1e4d9f5 is 50, key is test_row_0/A:col10/1732481331349/Put/seqid=0 2024-11-24T20:48:51,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742311_1487 (size=14741) 2024-11-24T20:48:51,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-24T20:48:51,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:51,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:51,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:51,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:51,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T20:48:51,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481391775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:51,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:51,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:51,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:51,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481391777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:51,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:51,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:51,797 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/5f218561758541c6b6ce0d598aa90d60 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/5f218561758541c6b6ce0d598aa90d60 2024-11-24T20:48:51,801 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/C of 94ec587900a8a4e84a03f1b687bbae49 into 5f218561758541c6b6ce0d598aa90d60(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:51,801 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:51,801 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/C, priority=12, startTime=1732481330952; duration=0sec 2024-11-24T20:48:51,801 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:51,801 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:C 2024-11-24T20:48:51,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:51,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481391885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:51,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:51,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:51,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:51,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481391885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:51,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:51,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:51,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:51,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:51,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-24T20:48:51,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:51,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:51,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:51,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:51,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:52,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-24T20:48:52,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:52,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:52,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:52,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:52,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:52,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:52,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T20:48:52,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:52,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481392094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:52,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:52,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481392095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:52,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:52,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:52,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:52,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:52,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:52,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:52,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/34bba7c371c8491381603161b1e4d9f5 2024-11-24T20:48:52,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/c889ee0e659b4e7c93ac8efe21e53550 is 50, key is test_row_0/B:col10/1732481331349/Put/seqid=0 2024-11-24T20:48:52,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742312_1488 (size=12301) 2024-11-24T20:48:52,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/c889ee0e659b4e7c93ac8efe21e53550 2024-11-24T20:48:52,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/053cf14ff24f4c828d5320dcdec62fe0 is 50, key is test_row_0/C:col10/1732481331349/Put/seqid=0 2024-11-24T20:48:52,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742313_1489 (size=12301) 2024-11-24T20:48:52,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-24T20:48:52,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:52,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:52,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:52,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:52,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:52,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-24T20:48:52,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:52,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:52,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:52,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:52,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:52,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481392405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:52,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481392405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:52,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:52,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:52,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:52,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:52,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:52,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:52,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-24T20:48:52,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:52,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:52,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:52,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:52,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:52,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/053cf14ff24f4c828d5320dcdec62fe0 2024-11-24T20:48:52,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/34bba7c371c8491381603161b1e4d9f5 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/34bba7c371c8491381603161b1e4d9f5 2024-11-24T20:48:52,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/34bba7c371c8491381603161b1e4d9f5, entries=200, sequenceid=365, filesize=14.4 K 2024-11-24T20:48:52,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/c889ee0e659b4e7c93ac8efe21e53550 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/c889ee0e659b4e7c93ac8efe21e53550 2024-11-24T20:48:52,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/c889ee0e659b4e7c93ac8efe21e53550, entries=150, sequenceid=365, filesize=12.0 K 2024-11-24T20:48:52,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/053cf14ff24f4c828d5320dcdec62fe0 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/053cf14ff24f4c828d5320dcdec62fe0 2024-11-24T20:48:52,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/053cf14ff24f4c828d5320dcdec62fe0, entries=150, sequenceid=365, filesize=12.0 K 2024-11-24T20:48:52,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 94ec587900a8a4e84a03f1b687bbae49 in 839ms, sequenceid=365, compaction requested=true 2024-11-24T20:48:52,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:52,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:48:52,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:52,583 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:52,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:48:52,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:52,583 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:52,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:48:52,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:52,584 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:52,584 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42567 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:52,584 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/B is initiating minor compaction (all files) 2024-11-24T20:48:52,584 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/A is initiating minor compaction (all files) 2024-11-24T20:48:52,584 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/B in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:52,584 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/A in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:52,584 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/9d7b1d1f9ba5435695440b4e722082a5, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/92f01fa196394e9f8ea91d3ce8f61497, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/c889ee0e659b4e7c93ac8efe21e53550] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=36.8 K 2024-11-24T20:48:52,584 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/b3de9083db63404ea043fab92f917b23, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/2e1b3474d572469c880798d6b26c62ed, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/34bba7c371c8491381603161b1e4d9f5] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=41.6 K 2024-11-24T20:48:52,584 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d7b1d1f9ba5435695440b4e722082a5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732481328925 2024-11-24T20:48:52,584 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3de9083db63404ea043fab92f917b23, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732481328925 2024-11-24T20:48:52,584 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 92f01fa196394e9f8ea91d3ce8f61497, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732481330124 2024-11-24T20:48:52,584 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e1b3474d572469c880798d6b26c62ed, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732481330124 2024-11-24T20:48:52,584 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting c889ee0e659b4e7c93ac8efe21e53550, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732481331349 2024-11-24T20:48:52,585 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34bba7c371c8491381603161b1e4d9f5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732481331349 2024-11-24T20:48:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T20:48:52,599 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#B#compaction#419 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:52,599 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#A#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:52,600 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/ef0eb88b2ed44f38938ccba0ebee0a01 is 50, key is test_row_0/B:col10/1732481331349/Put/seqid=0 2024-11-24T20:48:52,600 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/91ca28d7231a47cf8d214e941034436a is 50, key is test_row_0/A:col10/1732481331349/Put/seqid=0 2024-11-24T20:48:52,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742314_1490 (size=13187) 2024-11-24T20:48:52,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742315_1491 (size=13187) 2024-11-24T20:48:52,612 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/91ca28d7231a47cf8d214e941034436a as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/91ca28d7231a47cf8d214e941034436a 2024-11-24T20:48:52,617 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/A of 94ec587900a8a4e84a03f1b687bbae49 into 91ca28d7231a47cf8d214e941034436a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:52,617 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:52,617 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/A, priority=13, startTime=1732481332583; duration=0sec 2024-11-24T20:48:52,617 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:52,617 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:A 2024-11-24T20:48:52,617 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:52,618 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:52,618 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/C is initiating minor compaction (all files) 2024-11-24T20:48:52,618 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/C in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:52,618 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/5f218561758541c6b6ce0d598aa90d60, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/9de4aa80d731420c8faf78f40123ce07, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/053cf14ff24f4c828d5320dcdec62fe0] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=36.8 K 2024-11-24T20:48:52,618 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f218561758541c6b6ce0d598aa90d60, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732481328925 2024-11-24T20:48:52,618 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9de4aa80d731420c8faf78f40123ce07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732481330124 2024-11-24T20:48:52,619 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 053cf14ff24f4c828d5320dcdec62fe0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732481331349 2024-11-24T20:48:52,624 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#C#compaction#421 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:52,625 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/bd52d788f2f4414ca2373363f7bffb4a is 50, key is test_row_0/C:col10/1732481331349/Put/seqid=0 2024-11-24T20:48:52,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742316_1492 (size=13187) 2024-11-24T20:48:52,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-24T20:48:52,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:52,669 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 94ec587900a8a4e84a03f1b687bbae49 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-24T20:48:52,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/c759d8b91e8940fd87438a01cbe0d3b8 is 50, key is test_row_0/A:col10/1732481331773/Put/seqid=0 2024-11-24T20:48:52,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742317_1493 (size=12301) 2024-11-24T20:48:52,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:52,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:53,010 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/ef0eb88b2ed44f38938ccba0ebee0a01 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/ef0eb88b2ed44f38938ccba0ebee0a01 2024-11-24T20:48:53,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:53,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481393000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:53,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:53,014 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/B of 94ec587900a8a4e84a03f1b687bbae49 into ef0eb88b2ed44f38938ccba0ebee0a01(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:53,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:53,014 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:53,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:53,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481393001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:53,014 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/B, priority=13, startTime=1732481332583; duration=0sec 2024-11-24T20:48:53,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:53,014 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:53,014 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:B 2024-11-24T20:48:53,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:53,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:53,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:53,033 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/bd52d788f2f4414ca2373363f7bffb4a as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/bd52d788f2f4414ca2373363f7bffb4a 2024-11-24T20:48:53,036 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/C of 94ec587900a8a4e84a03f1b687bbae49 into bd52d788f2f4414ca2373363f7bffb4a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:53,036 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:53,036 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/C, priority=13, startTime=1732481332583; duration=0sec 2024-11-24T20:48:53,036 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:53,036 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:C 2024-11-24T20:48:53,076 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/c759d8b91e8940fd87438a01cbe0d3b8 2024-11-24T20:48:53,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/82149a43b7814c70a2f70dd16edf0ae5 is 50, key is test_row_0/B:col10/1732481331773/Put/seqid=0 2024-11-24T20:48:53,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742318_1494 (size=12301) 2024-11-24T20:48:53,085 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/82149a43b7814c70a2f70dd16edf0ae5 2024-11-24T20:48:53,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/b7c3269c1f16455fb6392f2fd0cc5087 is 50, key is test_row_0/C:col10/1732481331773/Put/seqid=0 2024-11-24T20:48:53,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742319_1495 (size=12301) 2024-11-24T20:48:53,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:53,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481393124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:53,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:53,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:53,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481393125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:53,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:53,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:53,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:53,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:53,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:53,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:53,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481393335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:53,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:53,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:53,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:53,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481393335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:53,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:53,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:53,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:53,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:53,496 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/b7c3269c1f16455fb6392f2fd0cc5087 2024-11-24T20:48:53,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/c759d8b91e8940fd87438a01cbe0d3b8 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/c759d8b91e8940fd87438a01cbe0d3b8 2024-11-24T20:48:53,557 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/c759d8b91e8940fd87438a01cbe0d3b8, entries=150, sequenceid=379, filesize=12.0 K 2024-11-24T20:48:53,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/82149a43b7814c70a2f70dd16edf0ae5 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/82149a43b7814c70a2f70dd16edf0ae5 2024-11-24T20:48:53,561 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/82149a43b7814c70a2f70dd16edf0ae5, entries=150, sequenceid=379, filesize=12.0 K 2024-11-24T20:48:53,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/b7c3269c1f16455fb6392f2fd0cc5087 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/b7c3269c1f16455fb6392f2fd0cc5087 2024-11-24T20:48:53,566 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/b7c3269c1f16455fb6392f2fd0cc5087, entries=150, sequenceid=379, filesize=12.0 K 2024-11-24T20:48:53,567 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 94ec587900a8a4e84a03f1b687bbae49 in 898ms, sequenceid=379, compaction requested=false 2024-11-24T20:48:53,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:53,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:53,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-24T20:48:53,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-24T20:48:53,573 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-24T20:48:53,573 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1220 sec 2024-11-24T20:48:53,575 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.1290 sec 2024-11-24T20:48:53,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T20:48:53,595 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:48:53,596 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:48:53,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-24T20:48:53,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T20:48:53,600 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:48:53,601 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:48:53,601 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:48:53,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:53,648 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:48:53,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94ec587900a8a4e84a03f1b687bbae49 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-24T20:48:53,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/87946bd74eb54becac500e89be41b9f8 is 50, key is test_row_0/A:col10/1732481333647/Put/seqid=0 2024-11-24T20:48:53,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742320_1496 (size=14741) 2024-11-24T20:48:53,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/87946bd74eb54becac500e89be41b9f8 2024-11-24T20:48:53,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/faa125da8bd844aaba29837ce9ab6d26 is 50, key is test_row_0/B:col10/1732481333647/Put/seqid=0 2024-11-24T20:48:53,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742321_1497 (size=12301) 2024-11-24T20:48:53,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:53,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481393679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:53,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:53,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:53,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:53,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:53,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481393687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:53,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:53,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:53,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:53,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T20:48:53,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T20:48:53,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:53,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:53,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:53,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:53,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:53,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:53,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:53,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481393795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:53,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:53,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:53,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:53,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:53,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481393805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:53,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:53,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:53,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:53,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T20:48:53,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:53,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:53,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:53,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:53,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:53,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:53,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T20:48:54,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:54,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481394005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:54,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:54,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:54,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:54,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:54,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481394015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:54,020 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:54,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T20:48:54,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:54,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:54,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:54,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:54,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:54,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:54,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/faa125da8bd844aaba29837ce9ab6d26 2024-11-24T20:48:54,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/6bdec47983264093a2537bf31c746fed is 50, key is test_row_0/C:col10/1732481333647/Put/seqid=0 2024-11-24T20:48:54,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742322_1498 (size=12301) 2024-11-24T20:48:54,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T20:48:54,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:54,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:54,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:54,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:54,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T20:48:54,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481394315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:54,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:54,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:54,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:54,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481394325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:54,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:54,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:54,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:54,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T20:48:54,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:54,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:54,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:54,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:54,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:54,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/6bdec47983264093a2537bf31c746fed 2024-11-24T20:48:54,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/87946bd74eb54becac500e89be41b9f8 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/87946bd74eb54becac500e89be41b9f8 2024-11-24T20:48:54,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/87946bd74eb54becac500e89be41b9f8, entries=200, sequenceid=406, filesize=14.4 K 2024-11-24T20:48:54,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/faa125da8bd844aaba29837ce9ab6d26 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/faa125da8bd844aaba29837ce9ab6d26 2024-11-24T20:48:54,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/faa125da8bd844aaba29837ce9ab6d26, entries=150, sequenceid=406, filesize=12.0 K 2024-11-24T20:48:54,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/6bdec47983264093a2537bf31c746fed as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/6bdec47983264093a2537bf31c746fed 2024-11-24T20:48:54,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/6bdec47983264093a2537bf31c746fed, entries=150, sequenceid=406, filesize=12.0 K 2024-11-24T20:48:54,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 94ec587900a8a4e84a03f1b687bbae49 in 844ms, sequenceid=406, compaction requested=true 2024-11-24T20:48:54,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:54,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:48:54,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:54,492 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:54,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:48:54,492 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:54,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:54,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:48:54,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:54,493 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:54,493 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:54,493 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/B is initiating minor compaction (all files) 2024-11-24T20:48:54,493 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/A is initiating minor compaction (all files) 2024-11-24T20:48:54,493 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/B in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:54,493 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/A in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:54,493 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/ef0eb88b2ed44f38938ccba0ebee0a01, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/82149a43b7814c70a2f70dd16edf0ae5, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/faa125da8bd844aaba29837ce9ab6d26] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=36.9 K 2024-11-24T20:48:54,493 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/91ca28d7231a47cf8d214e941034436a, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/c759d8b91e8940fd87438a01cbe0d3b8, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/87946bd74eb54becac500e89be41b9f8] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=39.3 K 2024-11-24T20:48:54,494 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting ef0eb88b2ed44f38938ccba0ebee0a01, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732481331349 2024-11-24T20:48:54,494 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91ca28d7231a47cf8d214e941034436a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732481331349 2024-11-24T20:48:54,494 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 82149a43b7814c70a2f70dd16edf0ae5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732481331772 2024-11-24T20:48:54,494 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting c759d8b91e8940fd87438a01cbe0d3b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732481331772 2024-11-24T20:48:54,494 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87946bd74eb54becac500e89be41b9f8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1732481332979 2024-11-24T20:48:54,494 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting faa125da8bd844aaba29837ce9ab6d26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1732481332980 2024-11-24T20:48:54,507 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#A#compaction#428 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:54,507 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/111f06b4580843558fe2bd63a1b582e5 is 50, key is test_row_0/A:col10/1732481333647/Put/seqid=0 2024-11-24T20:48:54,508 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#B#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:54,509 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/52a8d2934a5b44cd8151be281cfd20f0 is 50, key is test_row_0/B:col10/1732481333647/Put/seqid=0 2024-11-24T20:48:54,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T20:48:54,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:54,519 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 94ec587900a8a4e84a03f1b687bbae49 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T20:48:54,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/e8fed1f127e4475eaaeca870d488fd51 is 50, key is test_row_0/A:col10/1732481333686/Put/seqid=0 2024-11-24T20:48:54,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742324_1500 (size=13289) 2024-11-24T20:48:54,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742323_1499 (size=13289) 2024-11-24T20:48:54,534 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/111f06b4580843558fe2bd63a1b582e5 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/111f06b4580843558fe2bd63a1b582e5 2024-11-24T20:48:54,539 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/A of 94ec587900a8a4e84a03f1b687bbae49 into 111f06b4580843558fe2bd63a1b582e5(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:54,539 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:54,539 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/A, priority=13, startTime=1732481334492; duration=0sec 2024-11-24T20:48:54,539 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:54,539 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:A 2024-11-24T20:48:54,539 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:54,540 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:54,540 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/C is initiating minor compaction (all files) 2024-11-24T20:48:54,540 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/C in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:54,540 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/bd52d788f2f4414ca2373363f7bffb4a, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/b7c3269c1f16455fb6392f2fd0cc5087, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/6bdec47983264093a2537bf31c746fed] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=36.9 K 2024-11-24T20:48:54,541 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd52d788f2f4414ca2373363f7bffb4a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732481331349 2024-11-24T20:48:54,541 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7c3269c1f16455fb6392f2fd0cc5087, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732481331772 2024-11-24T20:48:54,541 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bdec47983264093a2537bf31c746fed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1732481332980 2024-11-24T20:48:54,551 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#C#compaction#431 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:54,551 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/6b3e9bef34124afb9f818f6252005885 is 50, key is test_row_0/C:col10/1732481333647/Put/seqid=0 2024-11-24T20:48:54,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742325_1501 (size=12301) 2024-11-24T20:48:54,554 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/e8fed1f127e4475eaaeca870d488fd51 2024-11-24T20:48:54,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742326_1502 (size=13289) 2024-11-24T20:48:54,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/989efe13eb7a49368c8c5c54ea6dcc46 is 50, key is test_row_0/B:col10/1732481333686/Put/seqid=0 2024-11-24T20:48:54,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742327_1503 (size=12301) 2024-11-24T20:48:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T20:48:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:54,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:54,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481394915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:54,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:54,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:54,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:54,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481394917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:54,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:54,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:54,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:54,932 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/52a8d2934a5b44cd8151be281cfd20f0 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/52a8d2934a5b44cd8151be281cfd20f0 2024-11-24T20:48:54,935 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/B of 94ec587900a8a4e84a03f1b687bbae49 into 52a8d2934a5b44cd8151be281cfd20f0(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:54,935 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:54,935 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/B, priority=13, startTime=1732481334492; duration=0sec 2024-11-24T20:48:54,935 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:54,935 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:B 2024-11-24T20:48:54,961 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/6b3e9bef34124afb9f818f6252005885 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/6b3e9bef34124afb9f818f6252005885 2024-11-24T20:48:54,965 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/989efe13eb7a49368c8c5c54ea6dcc46 2024-11-24T20:48:54,965 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/C of 94ec587900a8a4e84a03f1b687bbae49 into 6b3e9bef34124afb9f818f6252005885(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:54,965 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:54,965 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/C, priority=13, startTime=1732481334492; duration=0sec 2024-11-24T20:48:54,965 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:54,965 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:C 2024-11-24T20:48:54,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/f325c954c6814bd581247fc3c246bfbb is 50, key is test_row_0/C:col10/1732481333686/Put/seqid=0 2024-11-24T20:48:54,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742328_1504 (size=12301) 2024-11-24T20:48:54,972 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/f325c954c6814bd581247fc3c246bfbb 2024-11-24T20:48:54,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/e8fed1f127e4475eaaeca870d488fd51 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/e8fed1f127e4475eaaeca870d488fd51 2024-11-24T20:48:54,978 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/e8fed1f127e4475eaaeca870d488fd51, entries=150, sequenceid=417, filesize=12.0 K 2024-11-24T20:48:54,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/989efe13eb7a49368c8c5c54ea6dcc46 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/989efe13eb7a49368c8c5c54ea6dcc46 2024-11-24T20:48:54,981 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/989efe13eb7a49368c8c5c54ea6dcc46, entries=150, sequenceid=417, filesize=12.0 K 2024-11-24T20:48:54,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/f325c954c6814bd581247fc3c246bfbb as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/f325c954c6814bd581247fc3c246bfbb 2024-11-24T20:48:54,984 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/f325c954c6814bd581247fc3c246bfbb, entries=150, sequenceid=417, filesize=12.0 K 2024-11-24T20:48:54,985 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 94ec587900a8a4e84a03f1b687bbae49 in 466ms, sequenceid=417, compaction requested=false 2024-11-24T20:48:54,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:54,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:54,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-24T20:48:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-24T20:48:54,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-24T20:48:54,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3850 sec 2024-11-24T20:48:54,989 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.3920 sec 2024-11-24T20:48:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:55,031 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:48:55,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94ec587900a8a4e84a03f1b687bbae49 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-24T20:48:55,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/c2b9aec4baee44b09ec729e5346ac027 is 50, key is test_row_0/A:col10/1732481334914/Put/seqid=0 2024-11-24T20:48:55,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742329_1505 (size=14741) 2024-11-24T20:48:55,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481395059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:55,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:55,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:55,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481395061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:55,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:55,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:55,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:55,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:55,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:55,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481395175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:55,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:55,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481395175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:55,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:55,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481395385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:55,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:55,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:55,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:55,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:55,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481395385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:55,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:55,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:55,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:55,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/c2b9aec4baee44b09ec729e5346ac027 2024-11-24T20:48:55,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/b0a4f2fbe4804fd48edd5684bc1e0864 is 50, key is test_row_0/B:col10/1732481334914/Put/seqid=0 2024-11-24T20:48:55,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742330_1506 (size=12301) 2024-11-24T20:48:55,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481395695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:55,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:55,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:55,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:55,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:55,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481395695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:55,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:55,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:55,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T20:48:55,746 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:48:55,746 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:48:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-24T20:48:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T20:48:55,748 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:48:55,748 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:48:55,748 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:48:55,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/b0a4f2fbe4804fd48edd5684bc1e0864 2024-11-24T20:48:55,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/8024b529fa2c4a8689065bdb62e1535f is 50, key is test_row_0/C:col10/1732481334914/Put/seqid=0 2024-11-24T20:48:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T20:48:55,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742331_1507 (size=12301) 2024-11-24T20:48:55,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/8024b529fa2c4a8689065bdb62e1535f 2024-11-24T20:48:55,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/c2b9aec4baee44b09ec729e5346ac027 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/c2b9aec4baee44b09ec729e5346ac027 2024-11-24T20:48:55,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/c2b9aec4baee44b09ec729e5346ac027, entries=200, sequenceid=446, filesize=14.4 K 2024-11-24T20:48:55,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/b0a4f2fbe4804fd48edd5684bc1e0864 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/b0a4f2fbe4804fd48edd5684bc1e0864 2024-11-24T20:48:55,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/b0a4f2fbe4804fd48edd5684bc1e0864, entries=150, sequenceid=446, filesize=12.0 K 2024-11-24T20:48:55,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/8024b529fa2c4a8689065bdb62e1535f as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/8024b529fa2c4a8689065bdb62e1535f 2024-11-24T20:48:55,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/8024b529fa2c4a8689065bdb62e1535f, entries=150, sequenceid=446, filesize=12.0 K 2024-11-24T20:48:55,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 94ec587900a8a4e84a03f1b687bbae49 in 842ms, sequenceid=446, compaction requested=true 2024-11-24T20:48:55,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:55,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:48:55,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:55,873 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:55,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:48:55,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:55,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:48:55,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T20:48:55,874 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:55,874 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/A is initiating minor compaction (all files) 2024-11-24T20:48:55,874 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/A in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:55,874 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:55,874 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/111f06b4580843558fe2bd63a1b582e5, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/e8fed1f127e4475eaaeca870d488fd51, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/c2b9aec4baee44b09ec729e5346ac027] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=39.4 K 2024-11-24T20:48:55,875 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 111f06b4580843558fe2bd63a1b582e5, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1732481332980 2024-11-24T20:48:55,875 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8fed1f127e4475eaaeca870d488fd51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1732481333676 2024-11-24T20:48:55,875 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:55,875 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/B is initiating minor compaction (all files) 2024-11-24T20:48:55,875 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/B in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:55,875 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2b9aec4baee44b09ec729e5346ac027, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732481334909 2024-11-24T20:48:55,875 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/52a8d2934a5b44cd8151be281cfd20f0, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/989efe13eb7a49368c8c5c54ea6dcc46, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/b0a4f2fbe4804fd48edd5684bc1e0864] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=37.0 K 2024-11-24T20:48:55,875 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 52a8d2934a5b44cd8151be281cfd20f0, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1732481332980 2024-11-24T20:48:55,875 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 989efe13eb7a49368c8c5c54ea6dcc46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1732481333676 2024-11-24T20:48:55,876 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting b0a4f2fbe4804fd48edd5684bc1e0864, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732481334909 2024-11-24T20:48:55,881 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#A#compaction#437 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:55,882 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/fe459c487a0144538dfeff014be4da7a is 50, key is test_row_0/A:col10/1732481334914/Put/seqid=0 2024-11-24T20:48:55,882 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#B#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:55,883 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/490a7b2d83d44416a45a6d5e5e92bfe9 is 50, key is test_row_0/B:col10/1732481334914/Put/seqid=0 2024-11-24T20:48:55,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742332_1508 (size=13391) 2024-11-24T20:48:55,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742333_1509 (size=13391) 2024-11-24T20:48:55,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-24T20:48:55,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:55,900 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 94ec587900a8a4e84a03f1b687bbae49 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-24T20:48:55,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/4c9273abcdfb4b8aadfc8ac10476fac2 is 50, key is test_row_0/A:col10/1732481335058/Put/seqid=0 2024-11-24T20:48:55,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742334_1510 (size=9857) 2024-11-24T20:48:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T20:48:56,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:56,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:56,290 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/fe459c487a0144538dfeff014be4da7a as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/fe459c487a0144538dfeff014be4da7a 2024-11-24T20:48:56,293 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/A of 94ec587900a8a4e84a03f1b687bbae49 into fe459c487a0144538dfeff014be4da7a(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:56,293 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:56,293 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/A, priority=13, startTime=1732481335873; duration=0sec 2024-11-24T20:48:56,294 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:56,294 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:A 2024-11-24T20:48:56,294 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:56,294 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:56,294 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/C is initiating minor compaction (all files) 2024-11-24T20:48:56,294 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/C in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:56,294 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/6b3e9bef34124afb9f818f6252005885, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/f325c954c6814bd581247fc3c246bfbb, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/8024b529fa2c4a8689065bdb62e1535f] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=37.0 K 2024-11-24T20:48:56,295 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b3e9bef34124afb9f818f6252005885, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1732481332980 2024-11-24T20:48:56,295 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting f325c954c6814bd581247fc3c246bfbb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1732481333676 2024-11-24T20:48:56,295 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8024b529fa2c4a8689065bdb62e1535f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732481334909 2024-11-24T20:48:56,299 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/490a7b2d83d44416a45a6d5e5e92bfe9 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/490a7b2d83d44416a45a6d5e5e92bfe9 2024-11-24T20:48:56,302 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#C#compaction#440 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:56,303 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/658ae632169c4072b61029f3144faf9c is 50, key is test_row_0/C:col10/1732481334914/Put/seqid=0 2024-11-24T20:48:56,305 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/B of 94ec587900a8a4e84a03f1b687bbae49 into 490a7b2d83d44416a45a6d5e5e92bfe9(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:56,305 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:56,305 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/B, priority=13, startTime=1732481335873; duration=0sec 2024-11-24T20:48:56,305 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:56,305 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:B 2024-11-24T20:48:56,306 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/4c9273abcdfb4b8aadfc8ac10476fac2 2024-11-24T20:48:56,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742335_1511 (size=13391) 2024-11-24T20:48:56,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/2171bc4afb1b47f08b7fb1f39b054642 is 50, key is test_row_0/B:col10/1732481335058/Put/seqid=0 2024-11-24T20:48:56,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:56,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481396313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:56,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:56,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:56,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:56,321 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/658ae632169c4072b61029f3144faf9c as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/658ae632169c4072b61029f3144faf9c 2024-11-24T20:48:56,324 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/C of 94ec587900a8a4e84a03f1b687bbae49 into 658ae632169c4072b61029f3144faf9c(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:56,324 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:56,324 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/C, priority=13, startTime=1732481335873; duration=0sec 2024-11-24T20:48:56,324 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:56,324 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:C 2024-11-24T20:48:56,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742336_1512 (size=9857) 2024-11-24T20:48:56,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:56,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481396319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:56,327 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/2171bc4afb1b47f08b7fb1f39b054642 2024-11-24T20:48:56,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/d284be25327648879947c7ed747ea455 is 50, key is test_row_0/C:col10/1732481335058/Put/seqid=0 2024-11-24T20:48:56,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742337_1513 (size=9857) 2024-11-24T20:48:56,340 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/d284be25327648879947c7ed747ea455 2024-11-24T20:48:56,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/4c9273abcdfb4b8aadfc8ac10476fac2 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/4c9273abcdfb4b8aadfc8ac10476fac2 2024-11-24T20:48:56,348 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/4c9273abcdfb4b8aadfc8ac10476fac2, entries=100, sequenceid=456, filesize=9.6 K 2024-11-24T20:48:56,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/2171bc4afb1b47f08b7fb1f39b054642 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/2171bc4afb1b47f08b7fb1f39b054642 2024-11-24T20:48:56,351 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/2171bc4afb1b47f08b7fb1f39b054642, entries=100, sequenceid=456, filesize=9.6 K 2024-11-24T20:48:56,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/d284be25327648879947c7ed747ea455 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/d284be25327648879947c7ed747ea455 2024-11-24T20:48:56,354 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/d284be25327648879947c7ed747ea455, entries=100, sequenceid=456, filesize=9.6 K 2024-11-24T20:48:56,355 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 94ec587900a8a4e84a03f1b687bbae49 in 455ms, sequenceid=456, compaction requested=false 2024-11-24T20:48:56,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:56,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:56,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-24T20:48:56,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-24T20:48:56,359 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-24T20:48:56,359 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 609 msec 2024-11-24T20:48:56,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 614 msec 2024-11-24T20:48:56,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T20:48:56,375 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:48:56,375 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:48:56,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-24T20:48:56,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-24T20:48:56,377 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:48:56,378 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:48:56,378 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:48:56,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:56,430 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:48:56,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94ec587900a8a4e84a03f1b687bbae49 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-24T20:48:56,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/98d28871b11148aaafbb5878f43e3265 is 50, key is test_row_0/A:col10/1732481336429/Put/seqid=0 2024-11-24T20:48:56,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742338_1514 (size=14741) 2024-11-24T20:48:56,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:56,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481396452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:56,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:56,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:56,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:56,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:56,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481396457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:56,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:56,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:56,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:56,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-24T20:48:56,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-24T20:48:56,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:56,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:56,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:56,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:56,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:56,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:56,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:56,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481396565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:56,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:56,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481396575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:56,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:56,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:56,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:56,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-24T20:48:56,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:56,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:56,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:56,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:56,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:56,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:56,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-24T20:48:56,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:56,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481396774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:56,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:56,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:56,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:56,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:56,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481396785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:56,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:56,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:56,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:56,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:56,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51214 deadline: 1732481396815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:56,821 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_0' in 94ec587900a8a4e84a03f1b687bbae49 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18286 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:48:56,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:56,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:56,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:56,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:56,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51226 deadline: 1732481396825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:56,831 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_2' in 94ec587900a8a4e84a03f1b687bbae49 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18294 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:48:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:56,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-24T20:48:56,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:56,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:56,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:56,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:56,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:56,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:48:56,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/98d28871b11148aaafbb5878f43e3265 2024-11-24T20:48:56,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:56,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51228 deadline: 1732481396844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:56,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/80fc16a82cd14c4fb64ba023e8f65be7 is 50, key is test_row_0/B:col10/1732481336429/Put/seqid=0 2024-11-24T20:48:56,850 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_0' in 94ec587900a8a4e84a03f1b687bbae49 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18315 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:48:56,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:56,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:56,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:56,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742339_1515 (size=12301) 2024-11-24T20:48:56,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/80fc16a82cd14c4fb64ba023e8f65be7 2024-11-24T20:48:56,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/a02a08f918a74d8cb4b5e8056b9e9fed is 50, key is test_row_0/C:col10/1732481336429/Put/seqid=0 2024-11-24T20:48:56,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742340_1516 (size=12301) 2024-11-24T20:48:56,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/a02a08f918a74d8cb4b5e8056b9e9fed 2024-11-24T20:48:56,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/98d28871b11148aaafbb5878f43e3265 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/98d28871b11148aaafbb5878f43e3265 2024-11-24T20:48:56,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/98d28871b11148aaafbb5878f43e3265, entries=200, sequenceid=486, filesize=14.4 K 2024-11-24T20:48:56,886 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-24T20:48:56,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/80fc16a82cd14c4fb64ba023e8f65be7 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/80fc16a82cd14c4fb64ba023e8f65be7 2024-11-24T20:48:56,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/80fc16a82cd14c4fb64ba023e8f65be7, entries=150, sequenceid=486, filesize=12.0 K 2024-11-24T20:48:56,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/a02a08f918a74d8cb4b5e8056b9e9fed as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/a02a08f918a74d8cb4b5e8056b9e9fed 2024-11-24T20:48:56,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/a02a08f918a74d8cb4b5e8056b9e9fed, entries=150, sequenceid=486, filesize=12.0 K 2024-11-24T20:48:56,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 94ec587900a8a4e84a03f1b687bbae49 in 465ms, sequenceid=486, compaction requested=true 2024-11-24T20:48:56,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:56,896 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:56,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:48:56,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:56,896 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:56,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:48:56,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:56,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:48:56,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:56,897 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37989 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:56,897 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/A is initiating minor compaction (all files) 2024-11-24T20:48:56,897 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/A in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:56,897 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/fe459c487a0144538dfeff014be4da7a, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/4c9273abcdfb4b8aadfc8ac10476fac2, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/98d28871b11148aaafbb5878f43e3265] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=37.1 K 2024-11-24T20:48:56,897 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:56,897 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/B is initiating minor compaction (all files) 2024-11-24T20:48:56,897 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/B in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:56,897 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/490a7b2d83d44416a45a6d5e5e92bfe9, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/2171bc4afb1b47f08b7fb1f39b054642, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/80fc16a82cd14c4fb64ba023e8f65be7] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=34.7 K 2024-11-24T20:48:56,897 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe459c487a0144538dfeff014be4da7a, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732481334909 2024-11-24T20:48:56,897 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 490a7b2d83d44416a45a6d5e5e92bfe9, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732481334909 2024-11-24T20:48:56,897 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c9273abcdfb4b8aadfc8ac10476fac2, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1732481335058 2024-11-24T20:48:56,898 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 2171bc4afb1b47f08b7fb1f39b054642, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1732481335058 2024-11-24T20:48:56,898 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98d28871b11148aaafbb5878f43e3265, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1732481336297 2024-11-24T20:48:56,898 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 80fc16a82cd14c4fb64ba023e8f65be7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1732481336312 2024-11-24T20:48:56,916 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#A#compaction#446 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:56,916 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#B#compaction#447 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:56,916 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/9c9271c7c16d43cea9247adfe363d9db is 50, key is test_row_0/A:col10/1732481336429/Put/seqid=0 2024-11-24T20:48:56,919 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/ba8896b4725f4d518a6293eec574f00c is 50, key is test_row_0/B:col10/1732481336429/Put/seqid=0 2024-11-24T20:48:56,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742341_1517 (size=13493) 2024-11-24T20:48:56,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742342_1518 (size=13493) 2024-11-24T20:48:56,936 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/9c9271c7c16d43cea9247adfe363d9db as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/9c9271c7c16d43cea9247adfe363d9db 2024-11-24T20:48:56,936 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/ba8896b4725f4d518a6293eec574f00c as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/ba8896b4725f4d518a6293eec574f00c 2024-11-24T20:48:56,941 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/B of 94ec587900a8a4e84a03f1b687bbae49 into ba8896b4725f4d518a6293eec574f00c(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:56,941 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/A of 94ec587900a8a4e84a03f1b687bbae49 into 9c9271c7c16d43cea9247adfe363d9db(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:56,941 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:56,941 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:56,941 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/B, priority=13, startTime=1732481336896; duration=0sec 2024-11-24T20:48:56,941 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/A, priority=13, startTime=1732481336896; duration=0sec 2024-11-24T20:48:56,941 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:56,941 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:B 2024-11-24T20:48:56,942 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:56,942 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:A 2024-11-24T20:48:56,942 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:56,943 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:56,943 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/C is initiating minor compaction (all files) 2024-11-24T20:48:56,943 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/C in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:56,943 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/658ae632169c4072b61029f3144faf9c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/d284be25327648879947c7ed747ea455, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/a02a08f918a74d8cb4b5e8056b9e9fed] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=34.7 K 2024-11-24T20:48:56,943 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 658ae632169c4072b61029f3144faf9c, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732481334909 2024-11-24T20:48:56,944 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting d284be25327648879947c7ed747ea455, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1732481335058 2024-11-24T20:48:56,944 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting a02a08f918a74d8cb4b5e8056b9e9fed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1732481336312 2024-11-24T20:48:56,949 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#C#compaction#448 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:56,949 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/618e86c4296b49d4a77fc5eff23ae498 is 50, key is test_row_0/C:col10/1732481336429/Put/seqid=0 2024-11-24T20:48:56,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742343_1519 (size=13493) 2024-11-24T20:48:56,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-24T20:48:56,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:56,989 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 94ec587900a8a4e84a03f1b687bbae49 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-24T20:48:56,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/85b0866838c049d0906de3f264685baf is 50, key is test_row_0/A:col10/1732481336456/Put/seqid=0 2024-11-24T20:48:57,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742344_1520 (size=9857) 2024-11-24T20:48:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-24T20:48:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:57,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. as already flushing 2024-11-24T20:48:57,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481397154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:57,159 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:57,159 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:57,159 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:57,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481397158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:57,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:57,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:57,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:57,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481397266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:57,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:57,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:57,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:57,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481397275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:57,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:57,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:57,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:57,361 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/618e86c4296b49d4a77fc5eff23ae498 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/618e86c4296b49d4a77fc5eff23ae498 2024-11-24T20:48:57,366 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/C of 94ec587900a8a4e84a03f1b687bbae49 into 618e86c4296b49d4a77fc5eff23ae498(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:57,366 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:57,366 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/C, priority=13, startTime=1732481336896; duration=0sec 2024-11-24T20:48:57,366 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:57,367 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:C 2024-11-24T20:48:57,404 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/85b0866838c049d0906de3f264685baf 2024-11-24T20:48:57,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/6ac11258999e4d35b2abae96177063cb is 50, key is test_row_0/B:col10/1732481336456/Put/seqid=0 2024-11-24T20:48:57,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742345_1521 (size=9857) 2024-11-24T20:48:57,419 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/6ac11258999e4d35b2abae96177063cb 2024-11-24T20:48:57,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/169b710c92aa45af9c0a5ed9f04ddbd2 is 50, key is test_row_0/C:col10/1732481336456/Put/seqid=0 2024-11-24T20:48:57,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742346_1522 (size=9857) 2024-11-24T20:48:57,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481397475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:57,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:57,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:57,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:57,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481397485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:57,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:57,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:57,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:57,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-24T20:48:57,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:57,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51240 deadline: 1732481397785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:57,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:57,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:57,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:57,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:48:57,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51174 deadline: 1732481397795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:48:57,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 , the old value is region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:57,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:48:57,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., hostname=20b981ef5c27,34625,1732481182291, seqNum=2 because the exception is null or not the one we care about 2024-11-24T20:48:57,835 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/169b710c92aa45af9c0a5ed9f04ddbd2 2024-11-24T20:48:57,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/85b0866838c049d0906de3f264685baf as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/85b0866838c049d0906de3f264685baf 2024-11-24T20:48:57,841 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/85b0866838c049d0906de3f264685baf, entries=100, sequenceid=497, filesize=9.6 K 2024-11-24T20:48:57,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/6ac11258999e4d35b2abae96177063cb as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/6ac11258999e4d35b2abae96177063cb 2024-11-24T20:48:57,845 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/6ac11258999e4d35b2abae96177063cb, entries=100, sequenceid=497, filesize=9.6 K 2024-11-24T20:48:57,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/169b710c92aa45af9c0a5ed9f04ddbd2 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/169b710c92aa45af9c0a5ed9f04ddbd2 2024-11-24T20:48:57,848 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/169b710c92aa45af9c0a5ed9f04ddbd2, entries=100, sequenceid=497, filesize=9.6 K 2024-11-24T20:48:57,848 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 94ec587900a8a4e84a03f1b687bbae49 in 859ms, sequenceid=497, compaction requested=false 2024-11-24T20:48:57,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:57,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:57,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-24T20:48:57,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-24T20:48:57,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-24T20:48:57,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4720 sec 2024-11-24T20:48:57,852 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 1.4760 sec 2024-11-24T20:48:57,905 INFO [Thread-1890 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1890. 2024-11-24T20:48:57,905 DEBUG [Thread-1890 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:48:57,905 DEBUG [Thread-1890 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:57,906 DEBUG [Thread-1890 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:57,906 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:48:57,906 INFO [Thread-1884 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1884. 2024-11-24T20:48:57,906 DEBUG [Thread-1884 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:48:57,906 DEBUG [Thread-1884 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:57,906 DEBUG [Thread-1884 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:57,906 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:48:57,908 INFO [Thread-1882 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1882. 2024-11-24T20:48:57,908 DEBUG [Thread-1882 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:48:57,908 DEBUG [Thread-1882 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:57,908 DEBUG [Thread-1882 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:57,908 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:48:57,911 INFO [Thread-1888 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1888. 2024-11-24T20:48:57,911 DEBUG [Thread-1888 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:48:57,911 DEBUG [Thread-1888 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:57,911 DEBUG [Thread-1888 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:57,911 INFO [Thread-1886 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1886. 2024-11-24T20:48:57,911 DEBUG [Thread-1886 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:48:57,911 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:48:57,911 DEBUG [Thread-1886 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:57,911 DEBUG [Thread-1886 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:57,911 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:48:58,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:48:58,298 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:48:58,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94ec587900a8a4e84a03f1b687bbae49 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-24T20:48:58,298 INFO [Thread-1879 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1879. 2024-11-24T20:48:58,298 DEBUG [Thread-1879 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:48:58,298 DEBUG [Thread-1879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:58,298 DEBUG [Thread-1879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:58,299 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:48:58,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/eceb6270b67c4df9bcdc00b46d0d3e14 is 50, key is test_row_0/A:col10/1732481337124/Put/seqid=0 2024-11-24T20:48:58,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742347_1523 (size=12301) 2024-11-24T20:48:58,321 INFO [Thread-1873 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1873. 2024-11-24T20:48:58,321 DEBUG [Thread-1873 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:48:58,321 DEBUG [Thread-1873 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:58,321 DEBUG [Thread-1873 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:48:58,321 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:48:58,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-24T20:48:58,526 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:48:58,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/eceb6270b67c4df9bcdc00b46d0d3e14 2024-11-24T20:48:58,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/ac3df901475e4f8ba6131793f06e5169 is 50, key is test_row_0/B:col10/1732481337124/Put/seqid=0 2024-11-24T20:48:58,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742348_1524 (size=12301) 2024-11-24T20:48:59,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/ac3df901475e4f8ba6131793f06e5169 2024-11-24T20:48:59,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/83224f87fe6f43cf8276b5d01f1903eb is 50, key is test_row_0/C:col10/1732481337124/Put/seqid=0 2024-11-24T20:48:59,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742349_1525 (size=12301) 2024-11-24T20:48:59,527 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/83224f87fe6f43cf8276b5d01f1903eb 2024-11-24T20:48:59,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/eceb6270b67c4df9bcdc00b46d0d3e14 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/eceb6270b67c4df9bcdc00b46d0d3e14 2024-11-24T20:48:59,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/eceb6270b67c4df9bcdc00b46d0d3e14, entries=150, sequenceid=526, filesize=12.0 K 2024-11-24T20:48:59,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/ac3df901475e4f8ba6131793f06e5169 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/ac3df901475e4f8ba6131793f06e5169 2024-11-24T20:48:59,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/ac3df901475e4f8ba6131793f06e5169, entries=150, sequenceid=526, filesize=12.0 K 2024-11-24T20:48:59,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/83224f87fe6f43cf8276b5d01f1903eb as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/83224f87fe6f43cf8276b5d01f1903eb 2024-11-24T20:48:59,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/83224f87fe6f43cf8276b5d01f1903eb, entries=150, sequenceid=526, filesize=12.0 K 2024-11-24T20:48:59,541 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=6.71 KB/6870 for 94ec587900a8a4e84a03f1b687bbae49 in 1243ms, sequenceid=526, compaction requested=true 2024-11-24T20:48:59,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:59,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:48:59,542 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:59,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:59,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:48:59,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:59,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94ec587900a8a4e84a03f1b687bbae49:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:48:59,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:59,542 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:59,542 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35651 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:59,542 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/A is initiating minor compaction (all files) 2024-11-24T20:48:59,542 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35651 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:59,543 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/B is initiating minor compaction (all files) 2024-11-24T20:48:59,543 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/A in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:59,543 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/B in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:59,543 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/ba8896b4725f4d518a6293eec574f00c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/6ac11258999e4d35b2abae96177063cb, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/ac3df901475e4f8ba6131793f06e5169] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=34.8 K 2024-11-24T20:48:59,543 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/9c9271c7c16d43cea9247adfe363d9db, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/85b0866838c049d0906de3f264685baf, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/eceb6270b67c4df9bcdc00b46d0d3e14] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=34.8 K 2024-11-24T20:48:59,543 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c9271c7c16d43cea9247adfe363d9db, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1732481336312 2024-11-24T20:48:59,543 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting ba8896b4725f4d518a6293eec574f00c, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1732481336312 2024-11-24T20:48:59,543 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85b0866838c049d0906de3f264685baf, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1732481336450 2024-11-24T20:48:59,543 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ac11258999e4d35b2abae96177063cb, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1732481336450 2024-11-24T20:48:59,543 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting eceb6270b67c4df9bcdc00b46d0d3e14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1732481337124 2024-11-24T20:48:59,543 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting ac3df901475e4f8ba6131793f06e5169, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1732481337124 2024-11-24T20:48:59,549 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#B#compaction#455 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:59,549 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#A#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:59,549 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/1b24bcef449c4022bd7e157459ef1c62 is 50, key is test_row_0/B:col10/1732481337124/Put/seqid=0 2024-11-24T20:48:59,549 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/9982f928b7024e81b22e8d5aa992af07 is 50, key is test_row_0/A:col10/1732481337124/Put/seqid=0 2024-11-24T20:48:59,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742351_1527 (size=13595) 2024-11-24T20:48:59,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742350_1526 (size=13595) 2024-11-24T20:48:59,567 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/9982f928b7024e81b22e8d5aa992af07 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/9982f928b7024e81b22e8d5aa992af07 2024-11-24T20:48:59,571 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/A of 94ec587900a8a4e84a03f1b687bbae49 into 9982f928b7024e81b22e8d5aa992af07(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:59,571 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:59,571 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/A, priority=13, startTime=1732481339542; duration=0sec 2024-11-24T20:48:59,571 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:48:59,571 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:A 2024-11-24T20:48:59,571 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:48:59,572 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35651 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:48:59,572 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 94ec587900a8a4e84a03f1b687bbae49/C is initiating minor compaction (all files) 2024-11-24T20:48:59,572 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94ec587900a8a4e84a03f1b687bbae49/C in TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:48:59,573 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/618e86c4296b49d4a77fc5eff23ae498, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/169b710c92aa45af9c0a5ed9f04ddbd2, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/83224f87fe6f43cf8276b5d01f1903eb] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp, totalSize=34.8 K 2024-11-24T20:48:59,573 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 618e86c4296b49d4a77fc5eff23ae498, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1732481336312 2024-11-24T20:48:59,573 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 169b710c92aa45af9c0a5ed9f04ddbd2, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1732481336450 2024-11-24T20:48:59,573 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83224f87fe6f43cf8276b5d01f1903eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1732481337124 2024-11-24T20:48:59,579 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94ec587900a8a4e84a03f1b687bbae49#C#compaction#457 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:48:59,579 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/4fca436337e749198333d465a98898b1 is 50, key is test_row_0/C:col10/1732481337124/Put/seqid=0 2024-11-24T20:48:59,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742352_1528 (size=13595) 2024-11-24T20:48:59,592 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/4fca436337e749198333d465a98898b1 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/4fca436337e749198333d465a98898b1 2024-11-24T20:48:59,596 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/C of 94ec587900a8a4e84a03f1b687bbae49 into 4fca436337e749198333d465a98898b1(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:59,596 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:59,596 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/C, priority=13, startTime=1732481339542; duration=0sec 2024-11-24T20:48:59,597 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:59,597 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:C 2024-11-24T20:48:59,961 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/1b24bcef449c4022bd7e157459ef1c62 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/1b24bcef449c4022bd7e157459ef1c62 2024-11-24T20:48:59,964 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94ec587900a8a4e84a03f1b687bbae49/B of 94ec587900a8a4e84a03f1b687bbae49 into 1b24bcef449c4022bd7e157459ef1c62(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:48:59,964 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94ec587900a8a4e84a03f1b687bbae49: 2024-11-24T20:48:59,964 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., storeName=94ec587900a8a4e84a03f1b687bbae49/B, priority=13, startTime=1732481339542; duration=0sec 2024-11-24T20:48:59,964 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:48:59,964 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94ec587900a8a4e84a03f1b687bbae49:B 2024-11-24T20:49:06,866 INFO [Thread-1877 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1877. 2024-11-24T20:49:06,866 DEBUG [Thread-1877 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:49:06,866 DEBUG [Thread-1877 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:06,866 DEBUG [Thread-1877 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:06,866 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:49:06,887 INFO [Thread-1875 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1875. 2024-11-24T20:49:06,887 INFO [Thread-1871 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1871. 2024-11-24T20:49:06,887 DEBUG [Thread-1875 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:49:06,887 DEBUG [Thread-1871 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:49:06,887 DEBUG [Thread-1875 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:06,887 DEBUG [Thread-1871 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:06,887 DEBUG [Thread-1875 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:06,887 DEBUG [Thread-1871 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:06,887 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:49:06,887 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:49:06,887 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T20:49:06,887 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-24T20:49:06,888 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 180 2024-11-24T20:49:06,888 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-24T20:49:06,888 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-24T20:49:06,888 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 187 2024-11-24T20:49:06,888 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T20:49:06,888 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T20:49:06,888 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1515 2024-11-24T20:49:06,888 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4541 rows 2024-11-24T20:49:06,888 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1522 2024-11-24T20:49:06,888 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4566 rows 2024-11-24T20:49:06,888 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1514 2024-11-24T20:49:06,888 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4538 rows 2024-11-24T20:49:06,888 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1510 2024-11-24T20:49:06,888 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4523 rows 2024-11-24T20:49:06,888 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1507 2024-11-24T20:49:06,888 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4521 rows 2024-11-24T20:49:06,888 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T20:49:06,888 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T20:49:06,888 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:06,888 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:06,888 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:49:06,889 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T20:49:06,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T20:49:06,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-24T20:49:06,892 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732481346891"}]},"ts":"1732481346891"} 2024-11-24T20:49:06,893 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T20:49:06,893 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T20:49:06,893 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T20:49:06,895 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94ec587900a8a4e84a03f1b687bbae49, UNASSIGN}] 2024-11-24T20:49:06,895 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94ec587900a8a4e84a03f1b687bbae49, UNASSIGN 2024-11-24T20:49:06,896 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=94ec587900a8a4e84a03f1b687bbae49, regionState=CLOSING, regionLocation=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:06,897 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=94ec587900a8a4e84a03f1b687bbae49, UNASSIGN because future has completed 2024-11-24T20:49:06,897 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T20:49:06,897 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291}] 2024-11-24T20:49:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-24T20:49:07,050 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:49:07,050 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T20:49:07,050 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing 94ec587900a8a4e84a03f1b687bbae49, disabling compactions & flushes 2024-11-24T20:49:07,050 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:49:07,050 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:49:07,050 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. after waiting 0 ms 2024-11-24T20:49:07,050 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:49:07,050 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(2902): Flushing 94ec587900a8a4e84a03f1b687bbae49 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-24T20:49:07,057 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/45faf4fd8ef3407294bea9e48efbba9a is 50, key is test_row_0/A:col10/1732481346886/Put/seqid=0 2024-11-24T20:49:07,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742353_1529 (size=12301) 2024-11-24T20:49:07,062 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=536 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/45faf4fd8ef3407294bea9e48efbba9a 2024-11-24T20:49:07,072 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/2f869cc13d644711bdd3efcad736d74b is 50, key is test_row_0/B:col10/1732481346886/Put/seqid=0 2024-11-24T20:49:07,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742354_1530 (size=12301) 2024-11-24T20:49:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-24T20:49:07,478 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=536 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/2f869cc13d644711bdd3efcad736d74b 2024-11-24T20:49:07,492 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/031994832b6d4cbebd0651dfd542524a is 50, key is test_row_0/C:col10/1732481346886/Put/seqid=0 2024-11-24T20:49:07,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742355_1531 (size=12301) 2024-11-24T20:49:07,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-24T20:49:07,898 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=536 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/031994832b6d4cbebd0651dfd542524a 2024-11-24T20:49:07,909 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/A/45faf4fd8ef3407294bea9e48efbba9a as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/45faf4fd8ef3407294bea9e48efbba9a 2024-11-24T20:49:07,912 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/45faf4fd8ef3407294bea9e48efbba9a, entries=150, sequenceid=536, filesize=12.0 K 2024-11-24T20:49:07,913 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/B/2f869cc13d644711bdd3efcad736d74b as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/2f869cc13d644711bdd3efcad736d74b 2024-11-24T20:49:07,917 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/2f869cc13d644711bdd3efcad736d74b, entries=150, sequenceid=536, filesize=12.0 K 2024-11-24T20:49:07,917 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/.tmp/C/031994832b6d4cbebd0651dfd542524a as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/031994832b6d4cbebd0651dfd542524a 2024-11-24T20:49:07,921 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/031994832b6d4cbebd0651dfd542524a, entries=150, sequenceid=536, filesize=12.0 K 2024-11-24T20:49:07,921 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 94ec587900a8a4e84a03f1b687bbae49 in 871ms, sequenceid=536, compaction requested=false 2024-11-24T20:49:07,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/9d7c0cefd6064654a44eb23671aaff9d, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/535f9ae0b00c4914bc0fc9743c59995f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/15f24effc8584d219a5e0a74e6f27540, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/dc78a5ae1aa34e99a9c7f5db591906bb, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/23c4c832b1644fb68a457d6ade83b583, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/3b48c409a3c64c79988f11094bdd91c0, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/28f34beb11c248ffba737881342d4af1, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/87657b74b75e4f1c8cce8cd89d38ec94, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/ac65fa7f5bcd4df2a5223744718cf7d4, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/1231c89e5d6f499a92190db3eea5cbcd, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/780a9c774a8341988fa4ed65a051f7c7, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/a71bb620920a4cf9ae81aa62dfff465b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/9d99bd2223a146da8ce92b8b175e540b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/dce8023b56cb45adaaf5fd5090ffee07, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/51e907bc14e94cc2a870e393ae6a226e, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/f7686677454e4a198d43aff48327e783, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/9f37a066ca78440e924162f6b7026ecd, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/0b2b93ee8d9841f681247ab8451f2901, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/b018486889cf458eb6292e2c1489d42a, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/ff5ccb90be9b44afbae1a7ba7e34cbb4, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/fbc20bfcf68445369e77da12c8ff20bf, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/0d1f013ee541455d8413e52b86992506, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/0bcee7dd7397446ca90d9da3da78c5a5, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/b3de9083db63404ea043fab92f917b23, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/2e1b3474d572469c880798d6b26c62ed, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/34bba7c371c8491381603161b1e4d9f5, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/91ca28d7231a47cf8d214e941034436a, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/c759d8b91e8940fd87438a01cbe0d3b8, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/87946bd74eb54becac500e89be41b9f8, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/111f06b4580843558fe2bd63a1b582e5, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/e8fed1f127e4475eaaeca870d488fd51, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/c2b9aec4baee44b09ec729e5346ac027, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/fe459c487a0144538dfeff014be4da7a, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/4c9273abcdfb4b8aadfc8ac10476fac2, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/98d28871b11148aaafbb5878f43e3265, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/9c9271c7c16d43cea9247adfe363d9db, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/85b0866838c049d0906de3f264685baf, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/eceb6270b67c4df9bcdc00b46d0d3e14] to archive 2024-11-24T20:49:07,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T20:49:07,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/9d7c0cefd6064654a44eb23671aaff9d to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/9d7c0cefd6064654a44eb23671aaff9d 2024-11-24T20:49:07,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/535f9ae0b00c4914bc0fc9743c59995f to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/535f9ae0b00c4914bc0fc9743c59995f 2024-11-24T20:49:07,928 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/15f24effc8584d219a5e0a74e6f27540 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/15f24effc8584d219a5e0a74e6f27540 2024-11-24T20:49:07,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/dc78a5ae1aa34e99a9c7f5db591906bb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/dc78a5ae1aa34e99a9c7f5db591906bb 2024-11-24T20:49:07,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/23c4c832b1644fb68a457d6ade83b583 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/23c4c832b1644fb68a457d6ade83b583 2024-11-24T20:49:07,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/3b48c409a3c64c79988f11094bdd91c0 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/3b48c409a3c64c79988f11094bdd91c0 2024-11-24T20:49:07,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/28f34beb11c248ffba737881342d4af1 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/28f34beb11c248ffba737881342d4af1 2024-11-24T20:49:07,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/87657b74b75e4f1c8cce8cd89d38ec94 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/87657b74b75e4f1c8cce8cd89d38ec94 2024-11-24T20:49:07,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/ac65fa7f5bcd4df2a5223744718cf7d4 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/ac65fa7f5bcd4df2a5223744718cf7d4 2024-11-24T20:49:07,938 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/1231c89e5d6f499a92190db3eea5cbcd to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/1231c89e5d6f499a92190db3eea5cbcd 2024-11-24T20:49:07,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/780a9c774a8341988fa4ed65a051f7c7 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/780a9c774a8341988fa4ed65a051f7c7 2024-11-24T20:49:07,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/a71bb620920a4cf9ae81aa62dfff465b to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/a71bb620920a4cf9ae81aa62dfff465b 2024-11-24T20:49:07,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/9d99bd2223a146da8ce92b8b175e540b to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/9d99bd2223a146da8ce92b8b175e540b 2024-11-24T20:49:07,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/dce8023b56cb45adaaf5fd5090ffee07 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/dce8023b56cb45adaaf5fd5090ffee07 2024-11-24T20:49:07,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/51e907bc14e94cc2a870e393ae6a226e to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/51e907bc14e94cc2a870e393ae6a226e 2024-11-24T20:49:07,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/f7686677454e4a198d43aff48327e783 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/f7686677454e4a198d43aff48327e783 2024-11-24T20:49:07,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/9f37a066ca78440e924162f6b7026ecd to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/9f37a066ca78440e924162f6b7026ecd 2024-11-24T20:49:07,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/0b2b93ee8d9841f681247ab8451f2901 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/0b2b93ee8d9841f681247ab8451f2901 2024-11-24T20:49:07,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/b018486889cf458eb6292e2c1489d42a to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/b018486889cf458eb6292e2c1489d42a 2024-11-24T20:49:07,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/ff5ccb90be9b44afbae1a7ba7e34cbb4 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/ff5ccb90be9b44afbae1a7ba7e34cbb4 2024-11-24T20:49:07,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/fbc20bfcf68445369e77da12c8ff20bf to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/fbc20bfcf68445369e77da12c8ff20bf 2024-11-24T20:49:07,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/0d1f013ee541455d8413e52b86992506 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/0d1f013ee541455d8413e52b86992506 2024-11-24T20:49:07,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/0bcee7dd7397446ca90d9da3da78c5a5 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/0bcee7dd7397446ca90d9da3da78c5a5 2024-11-24T20:49:07,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/b3de9083db63404ea043fab92f917b23 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/b3de9083db63404ea043fab92f917b23 2024-11-24T20:49:07,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/2e1b3474d572469c880798d6b26c62ed to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/2e1b3474d572469c880798d6b26c62ed 2024-11-24T20:49:07,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/34bba7c371c8491381603161b1e4d9f5 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/34bba7c371c8491381603161b1e4d9f5 2024-11-24T20:49:07,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/91ca28d7231a47cf8d214e941034436a to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/91ca28d7231a47cf8d214e941034436a 2024-11-24T20:49:07,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/c759d8b91e8940fd87438a01cbe0d3b8 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/c759d8b91e8940fd87438a01cbe0d3b8 2024-11-24T20:49:07,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/87946bd74eb54becac500e89be41b9f8 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/87946bd74eb54becac500e89be41b9f8 2024-11-24T20:49:07,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/111f06b4580843558fe2bd63a1b582e5 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/111f06b4580843558fe2bd63a1b582e5 2024-11-24T20:49:07,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/e8fed1f127e4475eaaeca870d488fd51 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/e8fed1f127e4475eaaeca870d488fd51 2024-11-24T20:49:07,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/c2b9aec4baee44b09ec729e5346ac027 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/c2b9aec4baee44b09ec729e5346ac027 2024-11-24T20:49:07,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/fe459c487a0144538dfeff014be4da7a to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/fe459c487a0144538dfeff014be4da7a 2024-11-24T20:49:07,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/4c9273abcdfb4b8aadfc8ac10476fac2 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/4c9273abcdfb4b8aadfc8ac10476fac2 2024-11-24T20:49:07,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/98d28871b11148aaafbb5878f43e3265 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/98d28871b11148aaafbb5878f43e3265 2024-11-24T20:49:07,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/9c9271c7c16d43cea9247adfe363d9db to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/9c9271c7c16d43cea9247adfe363d9db 2024-11-24T20:49:07,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/85b0866838c049d0906de3f264685baf to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/85b0866838c049d0906de3f264685baf 2024-11-24T20:49:07,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/eceb6270b67c4df9bcdc00b46d0d3e14 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/eceb6270b67c4df9bcdc00b46d0d3e14 2024-11-24T20:49:07,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/1b067ce9a2f9416b84686b7f7f19857b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/e5dc6681a9574d8b8c443f452b4ed0a6, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/ef0b517001964fa595d58f26a318dcb9, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/57d71bb319fb4e6ca75ae1b5c778c578, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/a1ca01cf7e8f4ae2a840efb1e85bbe1e, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/b10f6d1da31f497a98fd53463cce39d7, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/7f415bccf77b4174be241e7ad0e2d5ff, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/a39eb05eedcf42b5a6b82deddb07b700, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/fb185c86f8f54d6ba26c7207b7b1c21b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/af1254ac3a55496991387c79fe1800e8, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/42d92306bf314ff3b6917c7f8255dea2, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/06fe13e558934a538b85e5ae3776ed42, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/3d2a7f59cb484c598fe6c48ee7a58a93, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/63f19c2ae6e2457fa8bcad9a055f134f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/dda5a6eb75c3458daaef6bed44327285, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/1cd4a4de3a844d1d936b80d464142a57, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/6f88b495acaa4ce9aec614121c45f275, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/00abad3b91464e5ab93784e358e12fdc, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/3366ab9c30094e5faa6c02038f30d879, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/827cf1638dfc4ca7ab80c9c7bd9e8f00, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/412b3dbefbf04f0395eeeef67a060858, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/6547c5329f0749539cda6c2e1d053b4a, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/9d7b1d1f9ba5435695440b4e722082a5, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/933b662931bd4faf9aeb5dac6ed5046f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/92f01fa196394e9f8ea91d3ce8f61497, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/ef0eb88b2ed44f38938ccba0ebee0a01, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/c889ee0e659b4e7c93ac8efe21e53550, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/82149a43b7814c70a2f70dd16edf0ae5, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/52a8d2934a5b44cd8151be281cfd20f0, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/faa125da8bd844aaba29837ce9ab6d26, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/989efe13eb7a49368c8c5c54ea6dcc46, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/490a7b2d83d44416a45a6d5e5e92bfe9, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/b0a4f2fbe4804fd48edd5684bc1e0864, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/2171bc4afb1b47f08b7fb1f39b054642, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/ba8896b4725f4d518a6293eec574f00c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/80fc16a82cd14c4fb64ba023e8f65be7, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/6ac11258999e4d35b2abae96177063cb, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/ac3df901475e4f8ba6131793f06e5169] to archive 2024-11-24T20:49:07,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T20:49:07,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/1b067ce9a2f9416b84686b7f7f19857b to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/1b067ce9a2f9416b84686b7f7f19857b 2024-11-24T20:49:07,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/e5dc6681a9574d8b8c443f452b4ed0a6 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/e5dc6681a9574d8b8c443f452b4ed0a6 2024-11-24T20:49:07,968 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/ef0b517001964fa595d58f26a318dcb9 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/ef0b517001964fa595d58f26a318dcb9 2024-11-24T20:49:07,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/57d71bb319fb4e6ca75ae1b5c778c578 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/57d71bb319fb4e6ca75ae1b5c778c578 2024-11-24T20:49:07,970 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/a1ca01cf7e8f4ae2a840efb1e85bbe1e to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/a1ca01cf7e8f4ae2a840efb1e85bbe1e 2024-11-24T20:49:07,972 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/b10f6d1da31f497a98fd53463cce39d7 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/b10f6d1da31f497a98fd53463cce39d7 2024-11-24T20:49:07,973 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/7f415bccf77b4174be241e7ad0e2d5ff to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/7f415bccf77b4174be241e7ad0e2d5ff 2024-11-24T20:49:07,974 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/a39eb05eedcf42b5a6b82deddb07b700 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/a39eb05eedcf42b5a6b82deddb07b700 2024-11-24T20:49:07,975 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/fb185c86f8f54d6ba26c7207b7b1c21b to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/fb185c86f8f54d6ba26c7207b7b1c21b 2024-11-24T20:49:07,976 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/af1254ac3a55496991387c79fe1800e8 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/af1254ac3a55496991387c79fe1800e8 2024-11-24T20:49:07,977 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/42d92306bf314ff3b6917c7f8255dea2 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/42d92306bf314ff3b6917c7f8255dea2 2024-11-24T20:49:07,979 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/06fe13e558934a538b85e5ae3776ed42 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/06fe13e558934a538b85e5ae3776ed42 2024-11-24T20:49:07,980 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/3d2a7f59cb484c598fe6c48ee7a58a93 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/3d2a7f59cb484c598fe6c48ee7a58a93 2024-11-24T20:49:07,981 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/63f19c2ae6e2457fa8bcad9a055f134f to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/63f19c2ae6e2457fa8bcad9a055f134f 2024-11-24T20:49:07,982 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/dda5a6eb75c3458daaef6bed44327285 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/dda5a6eb75c3458daaef6bed44327285 2024-11-24T20:49:07,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/1cd4a4de3a844d1d936b80d464142a57 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/1cd4a4de3a844d1d936b80d464142a57 2024-11-24T20:49:07,984 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/6f88b495acaa4ce9aec614121c45f275 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/6f88b495acaa4ce9aec614121c45f275 2024-11-24T20:49:07,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/00abad3b91464e5ab93784e358e12fdc to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/00abad3b91464e5ab93784e358e12fdc 2024-11-24T20:49:07,986 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/3366ab9c30094e5faa6c02038f30d879 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/3366ab9c30094e5faa6c02038f30d879 2024-11-24T20:49:07,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/827cf1638dfc4ca7ab80c9c7bd9e8f00 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/827cf1638dfc4ca7ab80c9c7bd9e8f00 2024-11-24T20:49:07,988 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/412b3dbefbf04f0395eeeef67a060858 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/412b3dbefbf04f0395eeeef67a060858 2024-11-24T20:49:07,989 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/6547c5329f0749539cda6c2e1d053b4a to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/6547c5329f0749539cda6c2e1d053b4a 2024-11-24T20:49:07,990 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/9d7b1d1f9ba5435695440b4e722082a5 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/9d7b1d1f9ba5435695440b4e722082a5 2024-11-24T20:49:07,991 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/933b662931bd4faf9aeb5dac6ed5046f to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/933b662931bd4faf9aeb5dac6ed5046f 2024-11-24T20:49:07,991 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/92f01fa196394e9f8ea91d3ce8f61497 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/92f01fa196394e9f8ea91d3ce8f61497 2024-11-24T20:49:07,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/ef0eb88b2ed44f38938ccba0ebee0a01 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/ef0eb88b2ed44f38938ccba0ebee0a01 2024-11-24T20:49:07,993 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/c889ee0e659b4e7c93ac8efe21e53550 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/c889ee0e659b4e7c93ac8efe21e53550 2024-11-24T20:49:07,994 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/82149a43b7814c70a2f70dd16edf0ae5 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/82149a43b7814c70a2f70dd16edf0ae5 2024-11-24T20:49:07,995 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/52a8d2934a5b44cd8151be281cfd20f0 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/52a8d2934a5b44cd8151be281cfd20f0 2024-11-24T20:49:07,996 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/faa125da8bd844aaba29837ce9ab6d26 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/faa125da8bd844aaba29837ce9ab6d26 2024-11-24T20:49:07,997 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/989efe13eb7a49368c8c5c54ea6dcc46 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/989efe13eb7a49368c8c5c54ea6dcc46 2024-11-24T20:49:07,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/490a7b2d83d44416a45a6d5e5e92bfe9 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/490a7b2d83d44416a45a6d5e5e92bfe9 2024-11-24T20:49:07,999 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/b0a4f2fbe4804fd48edd5684bc1e0864 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/b0a4f2fbe4804fd48edd5684bc1e0864 2024-11-24T20:49:08,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/2171bc4afb1b47f08b7fb1f39b054642 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/2171bc4afb1b47f08b7fb1f39b054642 2024-11-24T20:49:08,002 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/ba8896b4725f4d518a6293eec574f00c to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/ba8896b4725f4d518a6293eec574f00c 2024-11-24T20:49:08,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/80fc16a82cd14c4fb64ba023e8f65be7 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/80fc16a82cd14c4fb64ba023e8f65be7 2024-11-24T20:49:08,004 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/6ac11258999e4d35b2abae96177063cb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/6ac11258999e4d35b2abae96177063cb 2024-11-24T20:49:08,005 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/ac3df901475e4f8ba6131793f06e5169 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/ac3df901475e4f8ba6131793f06e5169 2024-11-24T20:49:08,006 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/9dde0ffc3f4a4e19938224c3ebe09074, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/12396d7056534ebd91b747c4a7a539b1, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/75e9286d95e84d209e1d98d0fa753eed, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/9adcec31e13a484c9fccfc9959762064, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/b967c3d465d3417aaa30bf1723cc5b9a, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/f2a0b69d54ca43959f0f0dc32d9f0186, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/12604e9a4c4f4e8996eab86d70d78117, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/7f6207dd546c4e12a2bf7ed757d0c1b9, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/dd832261c2864a52b981cca1d7fa9701, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/54e88b2cbeea4852897a74e7e9f370ea, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/e7137814d4134479bfac9626b43e8e0c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/a66eda88dd484ca7be59112a563c8faf, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/d4930a21a9aa4209bda7adaed4e81021, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/71d8acbd8f704008ade84854ddaa42c1, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/4d3b013925c5485e810668c8f7b348b5, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/0cac906f64924b4abb15596ad45aff8d, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/eef4834e70db40fba36b949362e70ba9, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/33334a29518142f5b66105aa510f682b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/615be6ed8a484972aefa50975ea9d1c0, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/019def6faf8445c6b684ae7a75e7e6bc, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/9e48298c8e204f86a6760ff8839627ec, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/271206f4a0b24da892bd6fd7365a3c21, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/5f218561758541c6b6ce0d598aa90d60, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/e061b23fae3844f484c31e3ac8b6bbdd, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/9de4aa80d731420c8faf78f40123ce07, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/bd52d788f2f4414ca2373363f7bffb4a, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/053cf14ff24f4c828d5320dcdec62fe0, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/b7c3269c1f16455fb6392f2fd0cc5087, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/6b3e9bef34124afb9f818f6252005885, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/6bdec47983264093a2537bf31c746fed, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/f325c954c6814bd581247fc3c246bfbb, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/658ae632169c4072b61029f3144faf9c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/8024b529fa2c4a8689065bdb62e1535f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/d284be25327648879947c7ed747ea455, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/618e86c4296b49d4a77fc5eff23ae498, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/a02a08f918a74d8cb4b5e8056b9e9fed, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/169b710c92aa45af9c0a5ed9f04ddbd2, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/83224f87fe6f43cf8276b5d01f1903eb] to archive 2024-11-24T20:49:08,007 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T20:49:08,009 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/9dde0ffc3f4a4e19938224c3ebe09074 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/9dde0ffc3f4a4e19938224c3ebe09074 2024-11-24T20:49:08,010 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/12396d7056534ebd91b747c4a7a539b1 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/12396d7056534ebd91b747c4a7a539b1 2024-11-24T20:49:08,011 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/75e9286d95e84d209e1d98d0fa753eed to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/75e9286d95e84d209e1d98d0fa753eed 2024-11-24T20:49:08,012 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/9adcec31e13a484c9fccfc9959762064 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/9adcec31e13a484c9fccfc9959762064 2024-11-24T20:49:08,013 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/b967c3d465d3417aaa30bf1723cc5b9a to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/b967c3d465d3417aaa30bf1723cc5b9a 2024-11-24T20:49:08,014 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/f2a0b69d54ca43959f0f0dc32d9f0186 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/f2a0b69d54ca43959f0f0dc32d9f0186 2024-11-24T20:49:08,015 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/12604e9a4c4f4e8996eab86d70d78117 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/12604e9a4c4f4e8996eab86d70d78117 2024-11-24T20:49:08,017 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/7f6207dd546c4e12a2bf7ed757d0c1b9 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/7f6207dd546c4e12a2bf7ed757d0c1b9 2024-11-24T20:49:08,018 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/dd832261c2864a52b981cca1d7fa9701 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/dd832261c2864a52b981cca1d7fa9701 2024-11-24T20:49:08,019 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/54e88b2cbeea4852897a74e7e9f370ea to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/54e88b2cbeea4852897a74e7e9f370ea 2024-11-24T20:49:08,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/e7137814d4134479bfac9626b43e8e0c to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/e7137814d4134479bfac9626b43e8e0c 2024-11-24T20:49:08,021 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/a66eda88dd484ca7be59112a563c8faf to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/a66eda88dd484ca7be59112a563c8faf 2024-11-24T20:49:08,022 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/d4930a21a9aa4209bda7adaed4e81021 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/d4930a21a9aa4209bda7adaed4e81021 2024-11-24T20:49:08,023 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/71d8acbd8f704008ade84854ddaa42c1 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/71d8acbd8f704008ade84854ddaa42c1 2024-11-24T20:49:08,024 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/4d3b013925c5485e810668c8f7b348b5 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/4d3b013925c5485e810668c8f7b348b5 2024-11-24T20:49:08,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-24T20:49:08,026 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/0cac906f64924b4abb15596ad45aff8d to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/0cac906f64924b4abb15596ad45aff8d 2024-11-24T20:49:08,027 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/eef4834e70db40fba36b949362e70ba9 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/eef4834e70db40fba36b949362e70ba9 2024-11-24T20:49:08,028 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/33334a29518142f5b66105aa510f682b to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/33334a29518142f5b66105aa510f682b 2024-11-24T20:49:08,029 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/615be6ed8a484972aefa50975ea9d1c0 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/615be6ed8a484972aefa50975ea9d1c0 2024-11-24T20:49:08,030 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/019def6faf8445c6b684ae7a75e7e6bc to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/019def6faf8445c6b684ae7a75e7e6bc 2024-11-24T20:49:08,031 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/9e48298c8e204f86a6760ff8839627ec to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/9e48298c8e204f86a6760ff8839627ec 2024-11-24T20:49:08,032 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/271206f4a0b24da892bd6fd7365a3c21 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/271206f4a0b24da892bd6fd7365a3c21 2024-11-24T20:49:08,033 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/5f218561758541c6b6ce0d598aa90d60 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/5f218561758541c6b6ce0d598aa90d60 2024-11-24T20:49:08,034 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/e061b23fae3844f484c31e3ac8b6bbdd to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/e061b23fae3844f484c31e3ac8b6bbdd 2024-11-24T20:49:08,035 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/9de4aa80d731420c8faf78f40123ce07 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/9de4aa80d731420c8faf78f40123ce07 2024-11-24T20:49:08,036 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/bd52d788f2f4414ca2373363f7bffb4a to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/bd52d788f2f4414ca2373363f7bffb4a 2024-11-24T20:49:08,037 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/053cf14ff24f4c828d5320dcdec62fe0 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/053cf14ff24f4c828d5320dcdec62fe0 2024-11-24T20:49:08,038 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/b7c3269c1f16455fb6392f2fd0cc5087 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/b7c3269c1f16455fb6392f2fd0cc5087 2024-11-24T20:49:08,039 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/6b3e9bef34124afb9f818f6252005885 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/6b3e9bef34124afb9f818f6252005885 2024-11-24T20:49:08,040 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/6bdec47983264093a2537bf31c746fed to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/6bdec47983264093a2537bf31c746fed 2024-11-24T20:49:08,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/f325c954c6814bd581247fc3c246bfbb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/f325c954c6814bd581247fc3c246bfbb 2024-11-24T20:49:08,042 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/658ae632169c4072b61029f3144faf9c to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/658ae632169c4072b61029f3144faf9c 2024-11-24T20:49:08,042 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/8024b529fa2c4a8689065bdb62e1535f to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/8024b529fa2c4a8689065bdb62e1535f 2024-11-24T20:49:08,043 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/d284be25327648879947c7ed747ea455 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/d284be25327648879947c7ed747ea455 2024-11-24T20:49:08,044 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/618e86c4296b49d4a77fc5eff23ae498 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/618e86c4296b49d4a77fc5eff23ae498 2024-11-24T20:49:08,045 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/a02a08f918a74d8cb4b5e8056b9e9fed to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/a02a08f918a74d8cb4b5e8056b9e9fed 2024-11-24T20:49:08,046 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/169b710c92aa45af9c0a5ed9f04ddbd2 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/169b710c92aa45af9c0a5ed9f04ddbd2 2024-11-24T20:49:08,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/83224f87fe6f43cf8276b5d01f1903eb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/83224f87fe6f43cf8276b5d01f1903eb 2024-11-24T20:49:08,052 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/recovered.edits/539.seqid, newMaxSeqId=539, maxSeqId=1 2024-11-24T20:49:08,053 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. 2024-11-24T20:49:08,053 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for 94ec587900a8a4e84a03f1b687bbae49: Waiting for close lock at 1732481347050Running coprocessor pre-close hooks at 1732481347050Disabling compacts and flushes for region at 1732481347050Disabling writes for close at 1732481347050Obtaining lock to block concurrent updates at 1732481347050Preparing flush snapshotting stores in 94ec587900a8a4e84a03f1b687bbae49 at 1732481347050Finished memstore snapshotting TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732481347051 (+1 ms)Flushing stores of TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49. at 1732481347052 (+1 ms)Flushing 94ec587900a8a4e84a03f1b687bbae49/A: creating writer at 1732481347052Flushing 94ec587900a8a4e84a03f1b687bbae49/A: appending metadata at 1732481347056 (+4 ms)Flushing 94ec587900a8a4e84a03f1b687bbae49/A: closing flushed file at 1732481347056Flushing 94ec587900a8a4e84a03f1b687bbae49/B: creating writer at 1732481347068 (+12 ms)Flushing 94ec587900a8a4e84a03f1b687bbae49/B: appending metadata at 1732481347071 (+3 ms)Flushing 94ec587900a8a4e84a03f1b687bbae49/B: closing flushed file at 1732481347071Flushing 94ec587900a8a4e84a03f1b687bbae49/C: creating writer at 1732481347488 (+417 ms)Flushing 94ec587900a8a4e84a03f1b687bbae49/C: appending metadata at 1732481347492 (+4 ms)Flushing 94ec587900a8a4e84a03f1b687bbae49/C: closing flushed file at 1732481347492Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53db523c: reopening flushed file at 1732481347908 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a79e0de: reopening flushed file at 1732481347912 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f60e80c: reopening flushed file at 1732481347917 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 94ec587900a8a4e84a03f1b687bbae49 in 871ms, sequenceid=536, compaction requested=false at 1732481347921 (+4 ms)Writing region close event to WAL at 1732481348049 (+128 ms)Running coprocessor post-close hooks at 1732481348053 (+4 ms)Closed at 1732481348053 2024-11-24T20:49:08,055 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed 94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:49:08,056 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=94ec587900a8a4e84a03f1b687bbae49, regionState=CLOSED 2024-11-24T20:49:08,057 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 because future has completed 2024-11-24T20:49:08,060 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-24T20:49:08,060 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure 94ec587900a8a4e84a03f1b687bbae49, server=20b981ef5c27,34625,1732481182291 in 1.1610 sec 2024-11-24T20:49:08,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-24T20:49:08,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94ec587900a8a4e84a03f1b687bbae49, UNASSIGN in 1.1650 sec 2024-11-24T20:49:08,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-24T20:49:08,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1700 sec 2024-11-24T20:49:08,067 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732481348066"}]},"ts":"1732481348066"} 2024-11-24T20:49:08,069 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T20:49:08,069 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T20:49:08,071 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1800 sec 2024-11-24T20:49:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-24T20:49:09,036 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:49:09,038 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T20:49:09,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:49:09,080 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:49:09,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-24T20:49:09,081 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=145, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:49:09,084 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:49:09,088 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A, FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B, FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C, FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/recovered.edits] 2024-11-24T20:49:09,092 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/45faf4fd8ef3407294bea9e48efbba9a to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/45faf4fd8ef3407294bea9e48efbba9a 2024-11-24T20:49:09,094 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/9982f928b7024e81b22e8d5aa992af07 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/A/9982f928b7024e81b22e8d5aa992af07 2024-11-24T20:49:09,097 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/1b24bcef449c4022bd7e157459ef1c62 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/1b24bcef449c4022bd7e157459ef1c62 2024-11-24T20:49:09,099 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/2f869cc13d644711bdd3efcad736d74b to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/B/2f869cc13d644711bdd3efcad736d74b 2024-11-24T20:49:09,103 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/031994832b6d4cbebd0651dfd542524a to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/031994832b6d4cbebd0651dfd542524a 2024-11-24T20:49:09,106 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/4fca436337e749198333d465a98898b1 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/C/4fca436337e749198333d465a98898b1 2024-11-24T20:49:09,110 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/recovered.edits/539.seqid to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49/recovered.edits/539.seqid 2024-11-24T20:49:09,111 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/94ec587900a8a4e84a03f1b687bbae49 2024-11-24T20:49:09,111 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T20:49:09,116 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=145, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:49:09,119 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T20:49:09,121 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T20:49:09,122 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=145, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:49:09,123 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T20:49:09,123 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732481349123"}]},"ts":"9223372036854775807"} 2024-11-24T20:49:09,125 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T20:49:09,125 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 94ec587900a8a4e84a03f1b687bbae49, NAME => 'TestAcidGuarantees,,1732481315695.94ec587900a8a4e84a03f1b687bbae49.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T20:49:09,125 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T20:49:09,125 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732481349125"}]},"ts":"9223372036854775807"} 2024-11-24T20:49:09,127 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T20:49:09,128 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=145, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:49:09,129 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 90 msec 2024-11-24T20:49:09,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-24T20:49:09,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T20:49:09,192 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:49:09,203 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=236 (was 242), OpenFileDescriptor=445 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=469 (was 487), ProcessCount=11 (was 11), AvailableMemoryMB=9154 (was 9587) 2024-11-24T20:49:09,213 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=236, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=469, ProcessCount=11, AvailableMemoryMB=9154 2024-11-24T20:49:09,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T20:49:09,215 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T20:49:09,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T20:49:09,217 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T20:49:09,217 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:49:09,217 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 146 2024-11-24T20:49:09,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-24T20:49:09,218 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T20:49:09,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742356_1532 (size=959) 2024-11-24T20:49:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-24T20:49:09,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-24T20:49:09,626 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a 2024-11-24T20:49:09,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742357_1533 (size=53) 2024-11-24T20:49:09,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-24T20:49:10,031 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:49:10,031 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5eaa0424086a20a4fd6f784698d02aeb, disabling compactions & flushes 2024-11-24T20:49:10,031 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:10,032 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:10,032 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. after waiting 0 ms 2024-11-24T20:49:10,032 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:10,032 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:10,032 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5eaa0424086a20a4fd6f784698d02aeb: Waiting for close lock at 1732481350031Disabling compacts and flushes for region at 1732481350031Disabling writes for close at 1732481350032 (+1 ms)Writing region close event to WAL at 1732481350032Closed at 1732481350032 2024-11-24T20:49:10,033 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T20:49:10,033 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732481350033"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732481350033"}]},"ts":"1732481350033"} 2024-11-24T20:49:10,034 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T20:49:10,035 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T20:49:10,035 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732481350035"}]},"ts":"1732481350035"} 2024-11-24T20:49:10,036 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T20:49:10,037 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eaa0424086a20a4fd6f784698d02aeb, ASSIGN}] 2024-11-24T20:49:10,037 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eaa0424086a20a4fd6f784698d02aeb, ASSIGN 2024-11-24T20:49:10,038 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eaa0424086a20a4fd6f784698d02aeb, ASSIGN; state=OFFLINE, location=20b981ef5c27,34625,1732481182291; forceNewPlan=false, retain=false 2024-11-24T20:49:10,189 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=5eaa0424086a20a4fd6f784698d02aeb, regionState=OPENING, regionLocation=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:10,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eaa0424086a20a4fd6f784698d02aeb, ASSIGN because future has completed 2024-11-24T20:49:10,191 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291}] 2024-11-24T20:49:10,345 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:10,345 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7752): Opening region: {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} 2024-11-24T20:49:10,345 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:10,345 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:49:10,345 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7794): checking encryption for 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:10,345 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7797): checking classloading for 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:10,347 INFO [StoreOpener-5eaa0424086a20a4fd6f784698d02aeb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:10,348 INFO [StoreOpener-5eaa0424086a20a4fd6f784698d02aeb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5eaa0424086a20a4fd6f784698d02aeb columnFamilyName A 2024-11-24T20:49:10,348 DEBUG [StoreOpener-5eaa0424086a20a4fd6f784698d02aeb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:49:10,348 INFO [StoreOpener-5eaa0424086a20a4fd6f784698d02aeb-1 {}] regionserver.HStore(327): Store=5eaa0424086a20a4fd6f784698d02aeb/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:49:10,348 INFO [StoreOpener-5eaa0424086a20a4fd6f784698d02aeb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:10,349 INFO [StoreOpener-5eaa0424086a20a4fd6f784698d02aeb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5eaa0424086a20a4fd6f784698d02aeb columnFamilyName B 2024-11-24T20:49:10,349 DEBUG [StoreOpener-5eaa0424086a20a4fd6f784698d02aeb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:49:10,350 INFO [StoreOpener-5eaa0424086a20a4fd6f784698d02aeb-1 {}] regionserver.HStore(327): Store=5eaa0424086a20a4fd6f784698d02aeb/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:49:10,350 INFO [StoreOpener-5eaa0424086a20a4fd6f784698d02aeb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:10,351 INFO [StoreOpener-5eaa0424086a20a4fd6f784698d02aeb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5eaa0424086a20a4fd6f784698d02aeb columnFamilyName C 2024-11-24T20:49:10,351 DEBUG [StoreOpener-5eaa0424086a20a4fd6f784698d02aeb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:49:10,351 INFO [StoreOpener-5eaa0424086a20a4fd6f784698d02aeb-1 {}] regionserver.HStore(327): Store=5eaa0424086a20a4fd6f784698d02aeb/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:49:10,351 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1038): replaying wal for 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:10,352 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:10,352 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:10,353 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1048): stopping wal replay for 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:10,353 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1060): Cleaning up temporary data for 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:10,353 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T20:49:10,354 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1093): writing seq id for 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:10,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-24T20:49:10,356 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T20:49:10,356 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1114): Opened 5eaa0424086a20a4fd6f784698d02aeb; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67938605, jitterRate=0.012364104390144348}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T20:49:10,356 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:10,356 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1006): Region open journal for 5eaa0424086a20a4fd6f784698d02aeb: Running coprocessor pre-open hook at 1732481350345Writing region info on filesystem at 1732481350345Initializing all the Stores at 1732481350346 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481350346Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481350346Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481350346Cleaning up temporary data from old regions at 1732481350353 (+7 ms)Running coprocessor post-open hooks at 1732481350356 (+3 ms)Region opened successfully at 1732481350356 2024-11-24T20:49:10,357 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., pid=148, masterSystemTime=1732481350342 2024-11-24T20:49:10,359 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:10,359 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:10,359 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=5eaa0424086a20a4fd6f784698d02aeb, regionState=OPEN, openSeqNum=2, regionLocation=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:10,360 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 because future has completed 2024-11-24T20:49:10,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-24T20:49:10,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure 5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 in 170 msec 2024-11-24T20:49:10,365 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-24T20:49:10,365 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eaa0424086a20a4fd6f784698d02aeb, ASSIGN in 327 msec 2024-11-24T20:49:10,366 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T20:49:10,366 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732481350366"}]},"ts":"1732481350366"} 2024-11-24T20:49:10,367 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T20:49:10,368 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T20:49:10,369 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1530 sec 2024-11-24T20:49:11,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-24T20:49:11,366 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:49:11,367 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64082ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:11,367 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:49:11,367 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:49:11,368 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:49:11,368 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:49:11,368 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:49:11,369 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bafdca8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:11,369 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:49:11,369 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:49:11,369 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:11,370 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60668, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:49:11,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a159c1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:11,371 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:49:11,372 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:49:11,372 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:49:11,373 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59730, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:49:11,374 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T20:49:11,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 20b981ef5c27,38339,1732481181363 2024-11-24T20:49:11,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@65986aa5 2024-11-24T20:49:11,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T20:49:11,376 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60682, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T20:49:11,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T20:49:11,378 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T20:49:11,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-24T20:49:11,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742358_1534 (size=995) 2024-11-24T20:49:11,799 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-24T20:49:11,799 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-24T20:49:11,801 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T20:49:11,803 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eaa0424086a20a4fd6f784698d02aeb, REOPEN/MOVE}] 2024-11-24T20:49:11,804 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eaa0424086a20a4fd6f784698d02aeb, REOPEN/MOVE 2024-11-24T20:49:11,804 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=5eaa0424086a20a4fd6f784698d02aeb, regionState=CLOSING, regionLocation=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:11,805 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eaa0424086a20a4fd6f784698d02aeb, REOPEN/MOVE because future has completed 2024-11-24T20:49:11,805 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T20:49:11,806 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291}] 2024-11-24T20:49:11,958 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(122): Close 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:11,958 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T20:49:11,958 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1722): Closing 5eaa0424086a20a4fd6f784698d02aeb, disabling compactions & flushes 2024-11-24T20:49:11,958 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:11,958 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:11,958 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. after waiting 0 ms 2024-11-24T20:49:11,958 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:11,961 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T20:49:11,961 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:11,961 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1676): Region close journal for 5eaa0424086a20a4fd6f784698d02aeb: Waiting for close lock at 1732481351958Running coprocessor pre-close hooks at 1732481351958Disabling compacts and flushes for region at 1732481351958Disabling writes for close at 1732481351958Writing region close event to WAL at 1732481351958Running coprocessor post-close hooks at 1732481351961 (+3 ms)Closed at 1732481351961 2024-11-24T20:49:11,962 WARN [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegionServer(3299): Not adding moved region record: 5eaa0424086a20a4fd6f784698d02aeb to self. 2024-11-24T20:49:11,963 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(157): Closed 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:11,964 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=5eaa0424086a20a4fd6f784698d02aeb, regionState=CLOSED 2024-11-24T20:49:11,965 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 because future has completed 2024-11-24T20:49:11,967 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-24T20:49:11,967 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; CloseRegionProcedure 5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 in 161 msec 2024-11-24T20:49:11,968 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eaa0424086a20a4fd6f784698d02aeb, REOPEN/MOVE; state=CLOSED, location=20b981ef5c27,34625,1732481182291; forceNewPlan=false, retain=true 2024-11-24T20:49:12,118 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=5eaa0424086a20a4fd6f784698d02aeb, regionState=OPENING, regionLocation=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:12,120 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eaa0424086a20a4fd6f784698d02aeb, REOPEN/MOVE because future has completed 2024-11-24T20:49:12,120 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291}] 2024-11-24T20:49:12,275 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:12,275 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7752): Opening region: {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} 2024-11-24T20:49:12,275 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:12,275 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T20:49:12,275 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7794): checking encryption for 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:12,276 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7797): checking classloading for 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:12,278 INFO [StoreOpener-5eaa0424086a20a4fd6f784698d02aeb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:12,279 INFO [StoreOpener-5eaa0424086a20a4fd6f784698d02aeb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5eaa0424086a20a4fd6f784698d02aeb columnFamilyName A 2024-11-24T20:49:12,280 DEBUG [StoreOpener-5eaa0424086a20a4fd6f784698d02aeb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:49:12,280 INFO [StoreOpener-5eaa0424086a20a4fd6f784698d02aeb-1 {}] regionserver.HStore(327): Store=5eaa0424086a20a4fd6f784698d02aeb/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:49:12,281 INFO [StoreOpener-5eaa0424086a20a4fd6f784698d02aeb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:12,281 INFO [StoreOpener-5eaa0424086a20a4fd6f784698d02aeb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5eaa0424086a20a4fd6f784698d02aeb columnFamilyName B 2024-11-24T20:49:12,281 DEBUG [StoreOpener-5eaa0424086a20a4fd6f784698d02aeb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:49:12,282 INFO [StoreOpener-5eaa0424086a20a4fd6f784698d02aeb-1 {}] regionserver.HStore(327): Store=5eaa0424086a20a4fd6f784698d02aeb/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:49:12,282 INFO [StoreOpener-5eaa0424086a20a4fd6f784698d02aeb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:12,282 INFO [StoreOpener-5eaa0424086a20a4fd6f784698d02aeb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5eaa0424086a20a4fd6f784698d02aeb columnFamilyName C 2024-11-24T20:49:12,283 DEBUG [StoreOpener-5eaa0424086a20a4fd6f784698d02aeb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:49:12,283 INFO [StoreOpener-5eaa0424086a20a4fd6f784698d02aeb-1 {}] regionserver.HStore(327): Store=5eaa0424086a20a4fd6f784698d02aeb/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T20:49:12,283 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1038): replaying wal for 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:12,284 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:12,285 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:12,285 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1048): stopping wal replay for 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:12,286 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1060): Cleaning up temporary data for 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:12,286 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T20:49:12,287 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1093): writing seq id for 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:12,288 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1114): Opened 5eaa0424086a20a4fd6f784698d02aeb; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61277809, jitterRate=-0.08688949048519135}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T20:49:12,288 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:12,289 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1006): Region open journal for 5eaa0424086a20a4fd6f784698d02aeb: Running coprocessor pre-open hook at 1732481352276Writing region info on filesystem at 1732481352276Initializing all the Stores at 1732481352276Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481352276Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481352276Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732481352277 (+1 ms)Cleaning up temporary data from old regions at 1732481352286 (+9 ms)Running coprocessor post-open hooks at 1732481352288 (+2 ms)Region opened successfully at 1732481352289 (+1 ms) 2024-11-24T20:49:12,290 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., pid=153, masterSystemTime=1732481352272 2024-11-24T20:49:12,292 DEBUG [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:12,292 INFO [RS_OPEN_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:12,293 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=5eaa0424086a20a4fd6f784698d02aeb, regionState=OPEN, openSeqNum=5, regionLocation=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:12,294 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 because future has completed 2024-11-24T20:49:12,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=151 2024-11-24T20:49:12,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=151, state=SUCCESS, hasLock=false; OpenRegionProcedure 5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 in 175 msec 2024-11-24T20:49:12,299 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-24T20:49:12,299 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eaa0424086a20a4fd6f784698d02aeb, REOPEN/MOVE in 494 msec 2024-11-24T20:49:12,301 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-24T20:49:12,301 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 498 msec 2024-11-24T20:49:12,303 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 924 msec 2024-11-24T20:49:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-24T20:49:12,304 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:49:12,305 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5eb37c09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:12,305 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:49:12,305 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:49:12,306 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:49:12,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:49:12,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:49:12,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f2e098, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:12,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:49:12,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:49:12,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:12,307 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49616, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:49:12,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a089a8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:12,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45d415d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:12,308 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:49:12,309 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:49:12,309 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:49:12,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:49:12,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:49:12,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e86b876, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:12,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:49:12,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:49:12,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:12,311 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49630, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:49:12,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@194a649d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:12,312 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e839bfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:12,312 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:49:12,312 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:49:12,313 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:49:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:49:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:49:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@170d61d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:49:12,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:49:12,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:12,314 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49656, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:49:12,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27726133, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:12,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31b8df26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:12,316 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:49:12,316 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:49:12,317 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:49:12,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:49:12,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:49:12,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4163263e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:12,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:49:12,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:49:12,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:12,319 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49672, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:49:12,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ef58d6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:12,320 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8f9db7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:12,320 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:49:12,320 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:49:12,321 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:49:12,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:49:12,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:49:12,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74b21545, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:12,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:49:12,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:49:12,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:12,322 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49698, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:49:12,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5daf88d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:12,324 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e7631c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:12,324 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:49:12,324 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:49:12,325 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:49:12,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:49:12,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:49:12,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@120f7cfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:12,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:49:12,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:49:12,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:12,326 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49722, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:49:12,327 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e12adf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:12,328 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35b8ab27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:12,328 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:49:12,328 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:49:12,329 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:49:12,329 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:49:12,329 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:49:12,329 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5da76108, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:12,329 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:49:12,329 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:49:12,329 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:12,330 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49730, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:49:12,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@644129c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:12,332 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e1056f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:12,332 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:49:12,332 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:49:12,333 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:49:12,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:49:12,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:49:12,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5be595b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:12,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:49:12,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:49:12,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:12,334 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49744, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:49:12,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@664cc076, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:12,336 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@294719e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:12,336 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:49:12,336 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:49:12,337 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:49:12,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:49:12,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:49:12,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62c944dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:12,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:49:12,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:49:12,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:12,339 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49774, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:49:12,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10f6dce7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:12,340 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@691c2b05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:12,340 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 20b981ef5c27,38339,-1 for getting cluster id 2024-11-24T20:49:12,340 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T20:49:12,341 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '450c21d0-94cf-4eb1-984f-e674d08bba31' 2024-11-24T20:49:12,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T20:49:12,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "450c21d0-94cf-4eb1-984f-e674d08bba31" 2024-11-24T20:49:12,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@684b9b07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:12,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [20b981ef5c27,38339,-1] 2024-11-24T20:49:12,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T20:49:12,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:12,343 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49792, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T20:49:12,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1da69891, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T20:49:12,344 DEBUG [Thread-2407 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:49:12,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:49:12,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:49:12,346 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40538, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:49:12,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5] 2024-11-24T20:49:12,352 DEBUG [Thread-2413 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:49:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:49:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:49:12,354 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40544, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:49:12,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5] 2024-11-24T20:49:12,360 DEBUG [Thread-2402 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:49:12,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:49:12,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:49:12,362 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40558, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:49:12,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5] 2024-11-24T20:49:12,381 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:49:12,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5eaa0424086a20a4fd6f784698d02aeb 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T20:49:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:12,388 DEBUG [Thread-2415 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:49:12,389 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:49:12,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:49:12,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:49:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-24T20:49:12,391 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:49:12,391 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:49:12,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T20:49:12,392 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:49:12,392 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:49:12,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5] 2024-11-24T20:49:12,410 DEBUG [Thread-2404 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:49:12,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:49:12,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:49:12,412 DEBUG [Thread-2396 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:49:12,413 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40572, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:49:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:49:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:49:12,415 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40580, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:49:12,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5] 2024-11-24T20:49:12,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5] 2024-11-24T20:49:12,444 DEBUG [Thread-2409 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:49:12,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:49:12,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:49:12,447 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40592, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:49:12,448 DEBUG [Thread-2398 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:49:12,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:49:12,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5] 2024-11-24T20:49:12,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:49:12,450 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40600, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:49:12,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5] 2024-11-24T20:49:12,452 DEBUG [Thread-2400 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:49:12,452 DEBUG [Thread-2411 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T20:49:12,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:49:12,453 DEBUG [Thread-2411 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=20b981ef5c27,34625,1732481182291, seqNum=-1] 2024-11-24T20:49:12,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:49:12,454 DEBUG [Thread-2411 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T20:49:12,454 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40618, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:49:12,454 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40616, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T20:49:12,456 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5] 2024-11-24T20:49:12,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5] 2024-11-24T20:49:12,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411245d12f0778e2d4cfb8f01772d2624059f_5eaa0424086a20a4fd6f784698d02aeb is 50, key is test_row_0/A:col10/1732481352372/Put/seqid=0 2024-11-24T20:49:12,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742359_1535 (size=12154) 2024-11-24T20:49:12,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481412463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:12,467 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:49:12,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481412463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:12,467 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:12,467 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:12,467 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:12,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:12,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:12,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:12,470 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411245d12f0778e2d4cfb8f01772d2624059f_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245d12f0778e2d4cfb8f01772d2624059f_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:12,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481412466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:12,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:12,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:12,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:12,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481412467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:12,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:12,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:12,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:12,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481412467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:12,471 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:12,471 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/50c705f2924d4f07b3dc787dc9f495db, store: [table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:12,472 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:12,472 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:12,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/50c705f2924d4f07b3dc787dc9f495db is 175, key is test_row_0/A:col10/1732481352372/Put/seqid=0 2024-11-24T20:49:12,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742360_1536 (size=30955) 2024-11-24T20:49:12,484 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/50c705f2924d4f07b3dc787dc9f495db 2024-11-24T20:49:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T20:49:12,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/fdbb452481ba4f27af3327ac9c35998f is 50, key is test_row_0/B:col10/1732481352372/Put/seqid=0 2024-11-24T20:49:12,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-24T20:49:12,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:12,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:12,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:12,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:12,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:12,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742361_1537 (size=12001) 2024-11-24T20:49:12,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/fdbb452481ba4f27af3327ac9c35998f 2024-11-24T20:49:12,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481412575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:12,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481412575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:12,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481412575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:12,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:12,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:12,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:12,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:12,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481412575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:12,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:12,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:12,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:12,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:12,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481412575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:12,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:12,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:12,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:12,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/581f505a5a2e4688a750ba3410f2ab32 is 50, key is test_row_0/C:col10/1732481352372/Put/seqid=0 2024-11-24T20:49:12,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742362_1538 (size=12001) 2024-11-24T20:49:12,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-24T20:49:12,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:12,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:12,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:12,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:12,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:12,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:12,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T20:49:12,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:12,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481412784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:12,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:12,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:12,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481412785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:12,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481412785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:12,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:12,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:12,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481412785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:12,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481412785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:12,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:12,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:12,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:12,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:12,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:12,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:12,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-24T20:49:12,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:12,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:12,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:12,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:12,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:12,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:13,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-24T20:49:13,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:13,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:13,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:13,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:13,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:13,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:13,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T20:49:13,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/581f505a5a2e4688a750ba3410f2ab32 2024-11-24T20:49:13,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/50c705f2924d4f07b3dc787dc9f495db as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/50c705f2924d4f07b3dc787dc9f495db 2024-11-24T20:49:13,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/50c705f2924d4f07b3dc787dc9f495db, entries=150, sequenceid=15, filesize=30.2 K 2024-11-24T20:49:13,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/fdbb452481ba4f27af3327ac9c35998f as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/fdbb452481ba4f27af3327ac9c35998f 2024-11-24T20:49:13,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/fdbb452481ba4f27af3327ac9c35998f, entries=150, sequenceid=15, filesize=11.7 K 2024-11-24T20:49:13,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/581f505a5a2e4688a750ba3410f2ab32 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/581f505a5a2e4688a750ba3410f2ab32 2024-11-24T20:49:13,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/581f505a5a2e4688a750ba3410f2ab32, entries=150, sequenceid=15, filesize=11.7 K 2024-11-24T20:49:13,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 5eaa0424086a20a4fd6f784698d02aeb in 669ms, sequenceid=15, compaction requested=false 2024-11-24T20:49:13,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:13,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:13,097 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:49:13,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5eaa0424086a20a4fd6f784698d02aeb 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-24T20:49:13,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:13,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481413101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:13,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:13,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:13,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481413105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:13,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481413105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:13,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:13,110 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,110 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,110 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,110 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,110 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:13,110 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:13,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112447cdf12ddeb74944b679d6fac6e0d873_5eaa0424086a20a4fd6f784698d02aeb is 50, key is test_row_0/A:col10/1732481352465/Put/seqid=0 2024-11-24T20:49:13,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:13,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481413110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:13,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:13,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481413111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:13,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:13,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:13,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742363_1539 (size=12154) 2024-11-24T20:49:13,140 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:49:13,143 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112447cdf12ddeb74944b679d6fac6e0d873_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112447cdf12ddeb74944b679d6fac6e0d873_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:13,144 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/d53712b6979a473d8baa5a4a4a3dcbd2, store: [table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:13,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/d53712b6979a473d8baa5a4a4a3dcbd2 is 175, key is test_row_0/A:col10/1732481352465/Put/seqid=0 2024-11-24T20:49:13,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-24T20:49:13,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:13,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:13,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:13,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:13,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:13,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:13,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742364_1540 (size=30955) 2024-11-24T20:49:13,166 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/d53712b6979a473d8baa5a4a4a3dcbd2 2024-11-24T20:49:13,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/bca323f5ab0a4f9ba9bc401100bd8934 is 50, key is test_row_0/B:col10/1732481352465/Put/seqid=0 2024-11-24T20:49:13,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742365_1541 (size=12001) 2024-11-24T20:49:13,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:13,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481413215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:13,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:13,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481413215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:13,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:13,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481413215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:13,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:13,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481413224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:13,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:13,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:13,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481413224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:13,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:13,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-24T20:49:13,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:13,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:13,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:13,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:13,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:13,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:13,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:13,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481413425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:13,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:13,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:13,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481413425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:13,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:13,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:13,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481413425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:13,428 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,428 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,428 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:13,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:13,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481413434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:13,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:13,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:13,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481413435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:13,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:13,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-24T20:49:13,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:13,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:13,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:13,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:13,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:13,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:13,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T20:49:13,578 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T20:49:13,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/bca323f5ab0a4f9ba9bc401100bd8934 2024-11-24T20:49:13,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/57048206afe040f9b02d6705ea27e284 is 50, key is test_row_0/C:col10/1732481352465/Put/seqid=0 2024-11-24T20:49:13,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742366_1542 (size=12001) 2024-11-24T20:49:13,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-24T20:49:13,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:13,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:13,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:13,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:13,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:13,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:13,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:13,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481413735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:13,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481413735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:13,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481413735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:13,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:13,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481413745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:13,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:13,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481413745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:13,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:13,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:13,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:13,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-24T20:49:13,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:13,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:13,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:13,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:13,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:13,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:13,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-24T20:49:13,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:13,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:13,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:13,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:13,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:13,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:14,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/57048206afe040f9b02d6705ea27e284 2024-11-24T20:49:14,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/d53712b6979a473d8baa5a4a4a3dcbd2 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/d53712b6979a473d8baa5a4a4a3dcbd2 2024-11-24T20:49:14,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/d53712b6979a473d8baa5a4a4a3dcbd2, entries=150, sequenceid=42, filesize=30.2 K 2024-11-24T20:49:14,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/bca323f5ab0a4f9ba9bc401100bd8934 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/bca323f5ab0a4f9ba9bc401100bd8934 2024-11-24T20:49:14,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/bca323f5ab0a4f9ba9bc401100bd8934, entries=150, sequenceid=42, filesize=11.7 K 2024-11-24T20:49:14,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/57048206afe040f9b02d6705ea27e284 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/57048206afe040f9b02d6705ea27e284 2024-11-24T20:49:14,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/57048206afe040f9b02d6705ea27e284, entries=150, sequenceid=42, filesize=11.7 K 2024-11-24T20:49:14,036 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 5eaa0424086a20a4fd6f784698d02aeb in 939ms, sequenceid=42, compaction requested=false 2024-11-24T20:49:14,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:14,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-24T20:49:14,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:14,077 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 5eaa0424086a20a4fd6f784698d02aeb 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-24T20:49:14,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240f03a0224c554026ae7ce8035b9f8c96_5eaa0424086a20a4fd6f784698d02aeb is 50, key is test_row_0/A:col10/1732481353099/Put/seqid=0 2024-11-24T20:49:14,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742367_1543 (size=12154) 2024-11-24T20:49:14,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:49:14,109 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240f03a0224c554026ae7ce8035b9f8c96_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240f03a0224c554026ae7ce8035b9f8c96_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:14,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/97306ac4d7af49d2b9ac835692cb2a69, store: [table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:14,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/97306ac4d7af49d2b9ac835692cb2a69 is 175, key is test_row_0/A:col10/1732481353099/Put/seqid=0 2024-11-24T20:49:14,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742368_1544 (size=30955) 2024-11-24T20:49:14,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:14,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:14,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:14,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481414274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:14,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:14,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481414274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:14,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:14,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:14,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481414274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:14,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:14,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481414275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:14,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:14,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:14,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481414275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:14,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:14,280 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,280 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,280 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:14,280 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,280 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,281 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:14,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:14,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481414385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:14,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:14,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481414385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:14,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481414385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:14,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:14,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481414386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:14,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481414386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:14,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:14,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T20:49:14,543 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/97306ac4d7af49d2b9ac835692cb2a69 2024-11-24T20:49:14,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/0cc52863a9bd469884ac42428acebf38 is 50, key is test_row_0/B:col10/1732481353099/Put/seqid=0 2024-11-24T20:49:14,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742369_1545 (size=12001) 2024-11-24T20:49:14,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:14,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481414596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:14,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:14,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:14,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481414596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:14,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:14,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481414596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:14,599 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,599 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,599 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:14,599 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:14,599 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,599 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:14,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481414596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:14,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:14,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481414596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:14,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:14,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:14,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481414905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:14,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:14,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481414905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481414905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481414905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:14,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:14,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:14,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481414905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:14,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:14,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:14,965 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/0cc52863a9bd469884ac42428acebf38 2024-11-24T20:49:14,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/3c6529f9da90406e846e81dff7398983 is 50, key is test_row_0/C:col10/1732481353099/Put/seqid=0 2024-11-24T20:49:14,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742370_1546 (size=12001) 2024-11-24T20:49:14,986 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/3c6529f9da90406e846e81dff7398983 2024-11-24T20:49:14,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/97306ac4d7af49d2b9ac835692cb2a69 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/97306ac4d7af49d2b9ac835692cb2a69 2024-11-24T20:49:14,995 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/97306ac4d7af49d2b9ac835692cb2a69, entries=150, sequenceid=52, filesize=30.2 K 2024-11-24T20:49:14,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/0cc52863a9bd469884ac42428acebf38 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/0cc52863a9bd469884ac42428acebf38 2024-11-24T20:49:14,999 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/0cc52863a9bd469884ac42428acebf38, entries=150, sequenceid=52, filesize=11.7 K 2024-11-24T20:49:15,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/3c6529f9da90406e846e81dff7398983 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/3c6529f9da90406e846e81dff7398983 2024-11-24T20:49:15,003 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/3c6529f9da90406e846e81dff7398983, entries=150, sequenceid=52, filesize=11.7 K 2024-11-24T20:49:15,004 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 5eaa0424086a20a4fd6f784698d02aeb in 927ms, sequenceid=52, compaction requested=true 2024-11-24T20:49:15,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:15,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:15,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-24T20:49:15,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-24T20:49:15,007 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-24T20:49:15,007 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6140 sec 2024-11-24T20:49:15,010 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.6190 sec 2024-11-24T20:49:15,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:15,418 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:49:15,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5eaa0424086a20a4fd6f784698d02aeb 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-24T20:49:15,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481415421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:15,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:15,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:15,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:15,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481415421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:15,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:15,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:15,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:15,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481415421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:15,427 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:15,427 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:15,427 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:15,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112418b7f784043644fb9435d9e9843efb3c_5eaa0424086a20a4fd6f784698d02aeb is 50, key is test_row_0/A:col10/1732481355416/Put/seqid=0 2024-11-24T20:49:15,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:15,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481415425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:15,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:15,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:15,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:15,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:15,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481415426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:15,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:15,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:15,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:15,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742371_1547 (size=12154) 2024-11-24T20:49:15,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:15,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481415535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:15,537 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:15,537 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:15,537 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:15,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:15,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481415535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:15,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:15,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:15,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:15,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:15,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481415535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:15,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:15,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:15,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:15,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:15,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481415536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:15,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:15,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481415536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:15,540 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:15,540 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:15,540 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:15,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:15,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:15,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:15,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:15,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481415745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:15,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:15,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481415745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:15,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:15,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481415746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:15,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:15,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481415746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:15,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:15,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481415746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:15,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:15,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:15,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:15,837 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:49:15,842 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112418b7f784043644fb9435d9e9843efb3c_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112418b7f784043644fb9435d9e9843efb3c_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:15,843 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/81aab15de1be48d4b1998036cb19a7ae, store: [table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:15,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/81aab15de1be48d4b1998036cb19a7ae is 175, key is test_row_0/A:col10/1732481355416/Put/seqid=0 2024-11-24T20:49:15,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742372_1548 (size=30955) 2024-11-24T20:49:15,858 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/81aab15de1be48d4b1998036cb19a7ae 2024-11-24T20:49:15,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/ea66ae124a1f4176b3d6527ea32aaa22 is 50, key is test_row_0/B:col10/1732481355416/Put/seqid=0 2024-11-24T20:49:15,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742373_1549 (size=12001) 2024-11-24T20:49:15,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/ea66ae124a1f4176b3d6527ea32aaa22 2024-11-24T20:49:15,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/61d73e81ef1549058bb12a5bd68625ce is 50, key is test_row_0/C:col10/1732481355416/Put/seqid=0 2024-11-24T20:49:15,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742374_1550 (size=12001) 2024-11-24T20:49:15,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/61d73e81ef1549058bb12a5bd68625ce 2024-11-24T20:49:15,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/81aab15de1be48d4b1998036cb19a7ae as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/81aab15de1be48d4b1998036cb19a7ae 2024-11-24T20:49:15,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/81aab15de1be48d4b1998036cb19a7ae, entries=150, sequenceid=81, filesize=30.2 K 2024-11-24T20:49:15,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/ea66ae124a1f4176b3d6527ea32aaa22 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/ea66ae124a1f4176b3d6527ea32aaa22 2024-11-24T20:49:15,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/ea66ae124a1f4176b3d6527ea32aaa22, entries=150, sequenceid=81, filesize=11.7 K 2024-11-24T20:49:15,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/61d73e81ef1549058bb12a5bd68625ce as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/61d73e81ef1549058bb12a5bd68625ce 2024-11-24T20:49:15,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/61d73e81ef1549058bb12a5bd68625ce, entries=150, sequenceid=81, filesize=11.7 K 2024-11-24T20:49:15,893 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=33.54 KB/34350 for 5eaa0424086a20a4fd6f784698d02aeb in 475ms, sequenceid=81, compaction requested=true 2024-11-24T20:49:15,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:15,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaa0424086a20a4fd6f784698d02aeb:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:49:15,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:49:15,893 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:49:15,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaa0424086a20a4fd6f784698d02aeb:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:49:15,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:49:15,893 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:49:15,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaa0424086a20a4fd6f784698d02aeb:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:49:15,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:49:15,894 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 123820 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:49:15,894 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:49:15,894 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 5eaa0424086a20a4fd6f784698d02aeb/A is initiating minor compaction (all files) 2024-11-24T20:49:15,894 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 5eaa0424086a20a4fd6f784698d02aeb/B is initiating minor compaction (all files) 2024-11-24T20:49:15,894 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaa0424086a20a4fd6f784698d02aeb/A in TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:15,894 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaa0424086a20a4fd6f784698d02aeb/B in TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:15,894 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/50c705f2924d4f07b3dc787dc9f495db, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/d53712b6979a473d8baa5a4a4a3dcbd2, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/97306ac4d7af49d2b9ac835692cb2a69, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/81aab15de1be48d4b1998036cb19a7ae] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp, totalSize=120.9 K 2024-11-24T20:49:15,894 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/fdbb452481ba4f27af3327ac9c35998f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/bca323f5ab0a4f9ba9bc401100bd8934, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/0cc52863a9bd469884ac42428acebf38, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/ea66ae124a1f4176b3d6527ea32aaa22] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp, totalSize=46.9 K 2024-11-24T20:49:15,894 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:15,894 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. files: [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/50c705f2924d4f07b3dc787dc9f495db, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/d53712b6979a473d8baa5a4a4a3dcbd2, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/97306ac4d7af49d2b9ac835692cb2a69, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/81aab15de1be48d4b1998036cb19a7ae] 2024-11-24T20:49:15,894 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting fdbb452481ba4f27af3327ac9c35998f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732481352372 2024-11-24T20:49:15,895 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting bca323f5ab0a4f9ba9bc401100bd8934, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732481352464 2024-11-24T20:49:15,895 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50c705f2924d4f07b3dc787dc9f495db, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732481352372 2024-11-24T20:49:15,895 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cc52863a9bd469884ac42428acebf38, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732481353099 2024-11-24T20:49:15,895 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting d53712b6979a473d8baa5a4a4a3dcbd2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732481352464 2024-11-24T20:49:15,895 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting ea66ae124a1f4176b3d6527ea32aaa22, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732481354273 2024-11-24T20:49:15,895 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97306ac4d7af49d2b9ac835692cb2a69, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732481353099 2024-11-24T20:49:15,895 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81aab15de1be48d4b1998036cb19a7ae, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732481354273 2024-11-24T20:49:15,903 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaa0424086a20a4fd6f784698d02aeb#B#compaction#473 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:49:15,903 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/66023724b035402f848a2437976dc477 is 50, key is test_row_0/B:col10/1732481355416/Put/seqid=0 2024-11-24T20:49:15,904 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:15,909 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124ae3c5ba993b844c196e58aace5420f97_5eaa0424086a20a4fd6f784698d02aeb store=[table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:15,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742375_1551 (size=12139) 2024-11-24T20:49:15,911 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124ae3c5ba993b844c196e58aace5420f97_5eaa0424086a20a4fd6f784698d02aeb, store=[table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:15,912 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124ae3c5ba993b844c196e58aace5420f97_5eaa0424086a20a4fd6f784698d02aeb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:15,914 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/66023724b035402f848a2437976dc477 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/66023724b035402f848a2437976dc477 2024-11-24T20:49:15,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742376_1552 (size=4469) 2024-11-24T20:49:15,919 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5eaa0424086a20a4fd6f784698d02aeb/B of 5eaa0424086a20a4fd6f784698d02aeb into 66023724b035402f848a2437976dc477(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:49:15,919 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:15,919 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., storeName=5eaa0424086a20a4fd6f784698d02aeb/B, priority=12, startTime=1732481355893; duration=0sec 2024-11-24T20:49:15,919 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:49:15,919 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaa0424086a20a4fd6f784698d02aeb:B 2024-11-24T20:49:15,919 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:49:15,920 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:49:15,920 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 5eaa0424086a20a4fd6f784698d02aeb/C is initiating minor compaction (all files) 2024-11-24T20:49:15,920 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaa0424086a20a4fd6f784698d02aeb/C in TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:15,920 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/581f505a5a2e4688a750ba3410f2ab32, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/57048206afe040f9b02d6705ea27e284, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/3c6529f9da90406e846e81dff7398983, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/61d73e81ef1549058bb12a5bd68625ce] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp, totalSize=46.9 K 2024-11-24T20:49:15,921 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 581f505a5a2e4688a750ba3410f2ab32, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732481352372 2024-11-24T20:49:15,921 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 57048206afe040f9b02d6705ea27e284, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732481352464 2024-11-24T20:49:15,922 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c6529f9da90406e846e81dff7398983, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732481353099 2024-11-24T20:49:15,922 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 61d73e81ef1549058bb12a5bd68625ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732481354273 2024-11-24T20:49:15,929 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaa0424086a20a4fd6f784698d02aeb#C#compaction#475 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:49:15,929 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/dbe1d21541b144ff9c35497cb2228f65 is 50, key is test_row_0/C:col10/1732481355416/Put/seqid=0 2024-11-24T20:49:15,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742377_1553 (size=12139) 2024-11-24T20:49:16,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:16,058 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:49:16,059 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5eaa0424086a20a4fd6f784698d02aeb 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-24T20:49:16,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124450fcd18c8e14ca1a0902e84aa707691_5eaa0424086a20a4fd6f784698d02aeb is 50, key is test_row_0/A:col10/1732481356058/Put/seqid=0 2024-11-24T20:49:16,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742378_1554 (size=12154) 2024-11-24T20:49:16,070 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:49:16,074 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124450fcd18c8e14ca1a0902e84aa707691_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124450fcd18c8e14ca1a0902e84aa707691_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:16,075 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/3816d39cc27d4cfd8c3ca6861a69c203, store: [table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:16,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/3816d39cc27d4cfd8c3ca6861a69c203 is 175, key is test_row_0/A:col10/1732481356058/Put/seqid=0 2024-11-24T20:49:16,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742379_1555 (size=30955) 2024-11-24T20:49:16,086 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/3816d39cc27d4cfd8c3ca6861a69c203 2024-11-24T20:49:16,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:16,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481416085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:16,091 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,091 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,091 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:16,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/396e661b33a541a0952ff9c316e316bd is 50, key is test_row_0/B:col10/1732481356058/Put/seqid=0 2024-11-24T20:49:16,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742380_1556 (size=12001) 2024-11-24T20:49:16,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:16,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481416089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:16,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:16,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:16,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481416091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:16,096 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,096 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,096 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:16,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:16,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481416092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:16,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:16,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:16,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481416092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:16,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:16,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:16,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481416195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:16,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:16,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:16,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481416205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:16,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:16,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:16,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481416205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:16,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:16,207 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,207 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481416206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:16,208 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:16,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:16,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:16,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481416206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:16,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,208 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:16,316 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaa0424086a20a4fd6f784698d02aeb#A#compaction#474 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:49:16,317 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/4005cd0f55394ba0bc0680198dbe9323 is 175, key is test_row_0/A:col10/1732481355416/Put/seqid=0 2024-11-24T20:49:16,337 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/dbe1d21541b144ff9c35497cb2228f65 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/dbe1d21541b144ff9c35497cb2228f65 2024-11-24T20:49:16,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742381_1557 (size=31093) 2024-11-24T20:49:16,343 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5eaa0424086a20a4fd6f784698d02aeb/C of 5eaa0424086a20a4fd6f784698d02aeb into dbe1d21541b144ff9c35497cb2228f65(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:49:16,343 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:16,343 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., storeName=5eaa0424086a20a4fd6f784698d02aeb/C, priority=12, startTime=1732481355893; duration=0sec 2024-11-24T20:49:16,343 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:49:16,343 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaa0424086a20a4fd6f784698d02aeb:C 2024-11-24T20:49:16,347 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/4005cd0f55394ba0bc0680198dbe9323 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/4005cd0f55394ba0bc0680198dbe9323 2024-11-24T20:49:16,352 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5eaa0424086a20a4fd6f784698d02aeb/A of 5eaa0424086a20a4fd6f784698d02aeb into 4005cd0f55394ba0bc0680198dbe9323(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:49:16,352 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:16,352 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., storeName=5eaa0424086a20a4fd6f784698d02aeb/A, priority=12, startTime=1732481355893; duration=0sec 2024-11-24T20:49:16,353 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:49:16,353 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaa0424086a20a4fd6f784698d02aeb:A 2024-11-24T20:49:16,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:16,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481416405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:16,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:16,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481416415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:16,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:16,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481416415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:16,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:16,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:16,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:16,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481416415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:16,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:16,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:16,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481416415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:16,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:16,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/396e661b33a541a0952ff9c316e316bd 2024-11-24T20:49:16,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/94a1382d1ae54cbda1f4678a5dc3a186 is 50, key is test_row_0/C:col10/1732481356058/Put/seqid=0 2024-11-24T20:49:16,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742382_1558 (size=12001) 2024-11-24T20:49:16,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T20:49:16,545 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:49:16,545 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:49:16,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-24T20:49:16,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T20:49:16,547 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:49:16,547 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:49:16,547 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:49:16,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T20:49:16,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T20:49:16,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:16,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:16,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:16,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:16,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:16,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:16,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:16,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481416715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:16,717 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,717 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,717 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:16,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:16,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481416725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:16,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:16,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481416726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:16,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:16,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:16,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:16,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481416726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:16,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:16,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:16,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481416726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:16,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:16,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:16,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T20:49:16,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:16,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:16,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:16,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:16,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:16,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:16,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T20:49:16,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/94a1382d1ae54cbda1f4678a5dc3a186 2024-11-24T20:49:16,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/3816d39cc27d4cfd8c3ca6861a69c203 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/3816d39cc27d4cfd8c3ca6861a69c203 2024-11-24T20:49:16,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/3816d39cc27d4cfd8c3ca6861a69c203, entries=150, sequenceid=94, filesize=30.2 K 2024-11-24T20:49:16,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/396e661b33a541a0952ff9c316e316bd as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/396e661b33a541a0952ff9c316e316bd 2024-11-24T20:49:16,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/396e661b33a541a0952ff9c316e316bd, entries=150, sequenceid=94, filesize=11.7 K 2024-11-24T20:49:16,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/94a1382d1ae54cbda1f4678a5dc3a186 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/94a1382d1ae54cbda1f4678a5dc3a186 2024-11-24T20:49:16,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/94a1382d1ae54cbda1f4678a5dc3a186, entries=150, sequenceid=94, filesize=11.7 K 2024-11-24T20:49:16,916 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-24T20:49:16,917 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 5eaa0424086a20a4fd6f784698d02aeb in 859ms, sequenceid=94, compaction requested=false 2024-11-24T20:49:16,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:17,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T20:49:17,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:17,005 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 5eaa0424086a20a4fd6f784698d02aeb 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-24T20:49:17,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124fa8ecf85cb31433bb1eb177d589ee2bc_5eaa0424086a20a4fd6f784698d02aeb is 50, key is test_row_0/A:col10/1732481356089/Put/seqid=0 2024-11-24T20:49:17,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742383_1559 (size=12154) 2024-11-24T20:49:17,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T20:49:17,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:17,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:17,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481417239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:17,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481417240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481417241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:17,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:17,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481417240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:17,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:17,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:17,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:17,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481417245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:17,249 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,249 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,249 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:17,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:17,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481417355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:17,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:17,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:17,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481417355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:17,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:17,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:17,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481417355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:17,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:17,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:17,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481417356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:17,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:17,359 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:17,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481417356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:17,359 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,359 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,359 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:17,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:49:17,431 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124fa8ecf85cb31433bb1eb177d589ee2bc_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fa8ecf85cb31433bb1eb177d589ee2bc_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:17,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/c5a8a9213a7c4fa4a73ffc0cb77a4b78, store: [table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:17,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/c5a8a9213a7c4fa4a73ffc0cb77a4b78 is 175, key is test_row_0/A:col10/1732481356089/Put/seqid=0 2024-11-24T20:49:17,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742384_1560 (size=30955) 2024-11-24T20:49:17,443 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/c5a8a9213a7c4fa4a73ffc0cb77a4b78 2024-11-24T20:49:17,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/4636fdc347bf4af4ba525b1bd495d42e is 50, key is test_row_0/B:col10/1732481356089/Put/seqid=0 2024-11-24T20:49:17,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742385_1561 (size=12001) 2024-11-24T20:49:17,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:17,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481417565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:17,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:17,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481417565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:17,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:17,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:17,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481417566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:17,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:17,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:17,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:17,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481417566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:17,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481417566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,569 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:17,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T20:49:17,858 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/4636fdc347bf4af4ba525b1bd495d42e 2024-11-24T20:49:17,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/7a921cb32cb444a3b6def39d841eb185 is 50, key is test_row_0/C:col10/1732481356089/Put/seqid=0 2024-11-24T20:49:17,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481417875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:17,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:17,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481417875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:17,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481417875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:17,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:17,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:17,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481417875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:17,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:17,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:17,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481417875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:17,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:17,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:17,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:17,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742386_1562 (size=12001) 2024-11-24T20:49:17,900 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/7a921cb32cb444a3b6def39d841eb185 2024-11-24T20:49:17,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/c5a8a9213a7c4fa4a73ffc0cb77a4b78 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/c5a8a9213a7c4fa4a73ffc0cb77a4b78 2024-11-24T20:49:17,910 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/c5a8a9213a7c4fa4a73ffc0cb77a4b78, entries=150, sequenceid=121, filesize=30.2 K 2024-11-24T20:49:17,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/4636fdc347bf4af4ba525b1bd495d42e as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/4636fdc347bf4af4ba525b1bd495d42e 2024-11-24T20:49:17,924 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/4636fdc347bf4af4ba525b1bd495d42e, entries=150, sequenceid=121, filesize=11.7 K 2024-11-24T20:49:17,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/7a921cb32cb444a3b6def39d841eb185 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/7a921cb32cb444a3b6def39d841eb185 2024-11-24T20:49:17,927 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/7a921cb32cb444a3b6def39d841eb185, entries=150, sequenceid=121, filesize=11.7 K 2024-11-24T20:49:17,929 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 5eaa0424086a20a4fd6f784698d02aeb in 924ms, sequenceid=121, compaction requested=true 2024-11-24T20:49:17,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:17,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:17,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-24T20:49:17,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-24T20:49:17,932 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-24T20:49:17,932 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3840 sec 2024-11-24T20:49:17,933 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.3860 sec 2024-11-24T20:49:18,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:18,387 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:49:18,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5eaa0424086a20a4fd6f784698d02aeb 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-24T20:49:18,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c3b08dadfef84b2ea9ad4bd0ffc195b6_5eaa0424086a20a4fd6f784698d02aeb is 50, key is test_row_0/A:col10/1732481357239/Put/seqid=0 2024-11-24T20:49:18,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742387_1563 (size=14744) 2024-11-24T20:49:18,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:18,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481418404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:18,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:18,410 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:18,410 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:18,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481418406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:18,410 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:18,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:18,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:18,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481418406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:18,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:18,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:18,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:18,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:18,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:18,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:18,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481418410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:18,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:18,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:18,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481418410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:18,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:18,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:18,414 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:18,414 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:18,414 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:18,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:18,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481418515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:18,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:18,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481418515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:18,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:18,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:18,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:18,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:18,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:18,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:18,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:18,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481418515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:18,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:18,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:18,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:18,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481418525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:18,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:18,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:18,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481418525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:18,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:18,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:18,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T20:49:18,695 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:49:18,696 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:49:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-24T20:49:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T20:49:18,698 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:49:18,699 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:49:18,699 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:49:18,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481418725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:18,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481418725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:18,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:18,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:18,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:18,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:18,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:18,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481418725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:18,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:18,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:18,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:18,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:18,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481418735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:18,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:18,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481418735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:18,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:18,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:18,738 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:18,738 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:18,738 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:18,796 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:49:18,802 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c3b08dadfef84b2ea9ad4bd0ffc195b6_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c3b08dadfef84b2ea9ad4bd0ffc195b6_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:18,802 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/6c7d009d7cdc4369b718e64ef28a0982, store: [table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:18,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/6c7d009d7cdc4369b718e64ef28a0982 is 175, key is test_row_0/A:col10/1732481357239/Put/seqid=0 2024-11-24T20:49:18,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T20:49:18,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742388_1564 (size=39699) 2024-11-24T20:49:18,814 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/6c7d009d7cdc4369b718e64ef28a0982 2024-11-24T20:49:18,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/fd033b03afb44a5687983a496d6d8e1c is 50, key is test_row_0/B:col10/1732481357239/Put/seqid=0 2024-11-24T20:49:18,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742389_1565 (size=12101) 2024-11-24T20:49:18,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/fd033b03afb44a5687983a496d6d8e1c 2024-11-24T20:49:18,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/49c39a04a0ab4b66a6609de5f3c9fc70 is 50, key is test_row_0/C:col10/1732481357239/Put/seqid=0 2024-11-24T20:49:18,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742390_1566 (size=12101) 2024-11-24T20:49:18,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/49c39a04a0ab4b66a6609de5f3c9fc70 2024-11-24T20:49:18,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/6c7d009d7cdc4369b718e64ef28a0982 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/6c7d009d7cdc4369b718e64ef28a0982 2024-11-24T20:49:18,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/6c7d009d7cdc4369b718e64ef28a0982, entries=200, sequenceid=133, filesize=38.8 K 2024-11-24T20:49:18,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/fd033b03afb44a5687983a496d6d8e1c as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/fd033b03afb44a5687983a496d6d8e1c 2024-11-24T20:49:18,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T20:49:18,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:18,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:18,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:18,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:18,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:18,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/fd033b03afb44a5687983a496d6d8e1c, entries=150, sequenceid=133, filesize=11.8 K 2024-11-24T20:49:18,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/49c39a04a0ab4b66a6609de5f3c9fc70 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/49c39a04a0ab4b66a6609de5f3c9fc70 2024-11-24T20:49:18,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/49c39a04a0ab4b66a6609de5f3c9fc70, entries=150, sequenceid=133, filesize=11.8 K 2024-11-24T20:49:18,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 5eaa0424086a20a4fd6f784698d02aeb in 477ms, sequenceid=133, compaction requested=true 2024-11-24T20:49:18,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:18,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaa0424086a20a4fd6f784698d02aeb:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:49:18,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:49:18,865 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:49:18,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaa0424086a20a4fd6f784698d02aeb:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:49:18,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:49:18,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaa0424086a20a4fd6f784698d02aeb:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:49:18,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:49:18,865 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:49:18,866 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:49:18,866 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132702 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:49:18,866 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 5eaa0424086a20a4fd6f784698d02aeb/B is initiating minor compaction (all files) 2024-11-24T20:49:18,866 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 5eaa0424086a20a4fd6f784698d02aeb/A is initiating minor compaction (all files) 2024-11-24T20:49:18,866 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaa0424086a20a4fd6f784698d02aeb/B in TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:18,866 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/66023724b035402f848a2437976dc477, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/396e661b33a541a0952ff9c316e316bd, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/4636fdc347bf4af4ba525b1bd495d42e, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/fd033b03afb44a5687983a496d6d8e1c] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp, totalSize=47.1 K 2024-11-24T20:49:18,866 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaa0424086a20a4fd6f784698d02aeb/A in TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:18,866 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/4005cd0f55394ba0bc0680198dbe9323, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/3816d39cc27d4cfd8c3ca6861a69c203, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/c5a8a9213a7c4fa4a73ffc0cb77a4b78, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/6c7d009d7cdc4369b718e64ef28a0982] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp, totalSize=129.6 K 2024-11-24T20:49:18,866 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:18,866 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. files: [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/4005cd0f55394ba0bc0680198dbe9323, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/3816d39cc27d4cfd8c3ca6861a69c203, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/c5a8a9213a7c4fa4a73ffc0cb77a4b78, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/6c7d009d7cdc4369b718e64ef28a0982] 2024-11-24T20:49:18,875 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4005cd0f55394ba0bc0680198dbe9323, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732481354273 2024-11-24T20:49:18,875 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 66023724b035402f848a2437976dc477, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732481354273 2024-11-24T20:49:18,875 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3816d39cc27d4cfd8c3ca6861a69c203, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732481356057 2024-11-24T20:49:18,875 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 396e661b33a541a0952ff9c316e316bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732481356057 2024-11-24T20:49:18,876 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5a8a9213a7c4fa4a73ffc0cb77a4b78, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732481356087 2024-11-24T20:49:18,876 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 4636fdc347bf4af4ba525b1bd495d42e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732481356087 2024-11-24T20:49:18,876 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c7d009d7cdc4369b718e64ef28a0982, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732481357238 2024-11-24T20:49:18,876 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting fd033b03afb44a5687983a496d6d8e1c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732481357238 2024-11-24T20:49:18,883 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:18,885 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaa0424086a20a4fd6f784698d02aeb#B#compaction#485 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:49:18,886 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/5da84091243c472bbb9940c99aa6004e is 50, key is test_row_0/B:col10/1732481357239/Put/seqid=0 2024-11-24T20:49:18,889 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124951b8bb2ecaf4fa996ec73de79f5e189_5eaa0424086a20a4fd6f784698d02aeb store=[table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:18,890 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124951b8bb2ecaf4fa996ec73de79f5e189_5eaa0424086a20a4fd6f784698d02aeb, store=[table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:18,891 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124951b8bb2ecaf4fa996ec73de79f5e189_5eaa0424086a20a4fd6f784698d02aeb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:18,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742391_1567 (size=12375) 2024-11-24T20:49:18,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742392_1568 (size=4469) 2024-11-24T20:49:18,898 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaa0424086a20a4fd6f784698d02aeb#A#compaction#486 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:49:18,899 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/b93342339047408e82f7fb869aa435f4 is 175, key is test_row_0/A:col10/1732481357239/Put/seqid=0 2024-11-24T20:49:18,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742393_1569 (size=31329) 2024-11-24T20:49:18,911 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/b93342339047408e82f7fb869aa435f4 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/b93342339047408e82f7fb869aa435f4 2024-11-24T20:49:18,918 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5eaa0424086a20a4fd6f784698d02aeb/A of 5eaa0424086a20a4fd6f784698d02aeb into b93342339047408e82f7fb869aa435f4(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:49:18,918 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:18,918 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., storeName=5eaa0424086a20a4fd6f784698d02aeb/A, priority=12, startTime=1732481358865; duration=0sec 2024-11-24T20:49:18,918 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:49:18,918 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaa0424086a20a4fd6f784698d02aeb:A 2024-11-24T20:49:18,918 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T20:49:18,920 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T20:49:18,920 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 5eaa0424086a20a4fd6f784698d02aeb/C is initiating minor compaction (all files) 2024-11-24T20:49:18,920 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaa0424086a20a4fd6f784698d02aeb/C in TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:18,920 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/dbe1d21541b144ff9c35497cb2228f65, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/94a1382d1ae54cbda1f4678a5dc3a186, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/7a921cb32cb444a3b6def39d841eb185, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/49c39a04a0ab4b66a6609de5f3c9fc70] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp, totalSize=47.1 K 2024-11-24T20:49:18,921 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbe1d21541b144ff9c35497cb2228f65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732481354273 2024-11-24T20:49:18,921 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94a1382d1ae54cbda1f4678a5dc3a186, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732481356057 2024-11-24T20:49:18,921 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a921cb32cb444a3b6def39d841eb185, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732481356087 2024-11-24T20:49:18,921 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49c39a04a0ab4b66a6609de5f3c9fc70, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732481357238 2024-11-24T20:49:18,928 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaa0424086a20a4fd6f784698d02aeb#C#compaction#487 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:49:18,929 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/ddb44ad9d34d4d6e9eaecf002cb98c7f is 50, key is test_row_0/C:col10/1732481357239/Put/seqid=0 2024-11-24T20:49:18,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742394_1570 (size=12375) 2024-11-24T20:49:18,935 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/ddb44ad9d34d4d6e9eaecf002cb98c7f as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/ddb44ad9d34d4d6e9eaecf002cb98c7f 2024-11-24T20:49:18,938 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5eaa0424086a20a4fd6f784698d02aeb/C of 5eaa0424086a20a4fd6f784698d02aeb into ddb44ad9d34d4d6e9eaecf002cb98c7f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:49:18,938 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:18,938 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., storeName=5eaa0424086a20a4fd6f784698d02aeb/C, priority=12, startTime=1732481358865; duration=0sec 2024-11-24T20:49:18,938 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:49:18,938 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaa0424086a20a4fd6f784698d02aeb:C 2024-11-24T20:49:19,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T20:49:19,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:19,006 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 5eaa0424086a20a4fd6f784698d02aeb 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-24T20:49:19,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e9723e74272c4602a0565e3af48622a1_5eaa0424086a20a4fd6f784698d02aeb is 50, key is test_row_0/A:col10/1732481358404/Put/seqid=0 2024-11-24T20:49:19,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742395_1571 (size=12304) 2024-11-24T20:49:19,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:49:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T20:49:19,021 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e9723e74272c4602a0565e3af48622a1_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e9723e74272c4602a0565e3af48622a1_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:19,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/22968dc3c8734110b69e52867e015885, store: [table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:19,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/22968dc3c8734110b69e52867e015885 is 175, key is test_row_0/A:col10/1732481358404/Put/seqid=0 2024-11-24T20:49:19,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742396_1572 (size=31105) 2024-11-24T20:49:19,031 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/22968dc3c8734110b69e52867e015885 2024-11-24T20:49:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:19,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:19,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/df5a65b30dea46209d1da9589a8f05a7 is 50, key is test_row_0/B:col10/1732481358404/Put/seqid=0 2024-11-24T20:49:19,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742397_1573 (size=12151) 2024-11-24T20:49:19,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481419049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:19,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:19,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481419050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:19,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:19,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481419051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:19,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:19,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:19,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481419053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:19,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481419053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:19,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:19,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:19,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:19,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481419165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:19,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:19,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481419165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:19,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:19,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:19,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:19,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481419165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:19,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:19,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:19,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481419166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:19,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:19,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:19,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481419166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:19,169 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,169 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,169 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:19,300 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/5da84091243c472bbb9940c99aa6004e as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/5da84091243c472bbb9940c99aa6004e 2024-11-24T20:49:19,308 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5eaa0424086a20a4fd6f784698d02aeb/B of 5eaa0424086a20a4fd6f784698d02aeb into 5da84091243c472bbb9940c99aa6004e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:49:19,308 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:19,308 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., storeName=5eaa0424086a20a4fd6f784698d02aeb/B, priority=12, startTime=1732481358865; duration=0sec 2024-11-24T20:49:19,308 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:49:19,308 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaa0424086a20a4fd6f784698d02aeb:B 2024-11-24T20:49:19,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T20:49:19,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:19,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:19,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481419376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481419376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481419377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:19,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:19,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:19,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481419377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481419377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:19,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:19,379 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,379 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:19,449 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/df5a65b30dea46209d1da9589a8f05a7 2024-11-24T20:49:19,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/63e74b1d5cdb4e4c9a82ead7d3abf047 is 50, key is test_row_0/C:col10/1732481358404/Put/seqid=0 2024-11-24T20:49:19,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742398_1574 (size=12151) 2024-11-24T20:49:19,461 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/63e74b1d5cdb4e4c9a82ead7d3abf047 2024-11-24T20:49:19,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/22968dc3c8734110b69e52867e015885 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/22968dc3c8734110b69e52867e015885 2024-11-24T20:49:19,467 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/22968dc3c8734110b69e52867e015885, entries=150, sequenceid=159, filesize=30.4 K 2024-11-24T20:49:19,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/df5a65b30dea46209d1da9589a8f05a7 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/df5a65b30dea46209d1da9589a8f05a7 2024-11-24T20:49:19,471 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/df5a65b30dea46209d1da9589a8f05a7, entries=150, sequenceid=159, filesize=11.9 K 2024-11-24T20:49:19,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/63e74b1d5cdb4e4c9a82ead7d3abf047 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/63e74b1d5cdb4e4c9a82ead7d3abf047 2024-11-24T20:49:19,475 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/63e74b1d5cdb4e4c9a82ead7d3abf047, entries=150, sequenceid=159, filesize=11.9 K 2024-11-24T20:49:19,476 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 5eaa0424086a20a4fd6f784698d02aeb in 469ms, sequenceid=159, compaction requested=false 2024-11-24T20:49:19,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:19,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:19,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-24T20:49:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-24T20:49:19,478 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-24T20:49:19,478 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 778 msec 2024-11-24T20:49:19,480 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 783 msec 2024-11-24T20:49:19,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:19,688 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:49:19,688 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5eaa0424086a20a4fd6f784698d02aeb 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-24T20:49:19,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241de45f10ad2a409187cf2292c5fb7a77_5eaa0424086a20a4fd6f784698d02aeb is 50, key is test_row_0/A:col10/1732481359688/Put/seqid=0 2024-11-24T20:49:19,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:19,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481419720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:19,726 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,726 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,726 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:19,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742399_1575 (size=12304) 2024-11-24T20:49:19,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:19,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481419721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:19,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:19,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:19,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481419722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:19,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:19,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:19,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481419726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:19,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:19,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481419725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:19,733 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:49:19,736 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241de45f10ad2a409187cf2292c5fb7a77_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241de45f10ad2a409187cf2292c5fb7a77_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:19,737 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/7307b0dca80044fc8f30fac410bab605, store: [table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:19,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/7307b0dca80044fc8f30fac410bab605 is 175, key is test_row_0/A:col10/1732481359688/Put/seqid=0 2024-11-24T20:49:19,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742400_1576 (size=31105) 2024-11-24T20:49:19,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T20:49:19,835 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:49:19,835 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:49:19,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-24T20:49:19,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T20:49:19,837 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:49:19,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:19,837 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:49:19,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481419835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:19,837 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:49:19,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:19,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481419835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:19,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:19,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481419835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:19,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:19,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:19,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481419835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:19,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,838 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:19,838 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,838 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:19,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:19,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:19,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481419835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:19,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:19,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:19,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T20:49:19,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-24T20:49:19,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:19,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:19,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:19,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:19,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:19,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:20,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481420045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:20,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481420045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:20,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:20,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481420045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:20,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481420045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:20,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481420045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:20,141 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/7307b0dca80044fc8f30fac410bab605 2024-11-24T20:49:20,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-24T20:49:20,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:20,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:20,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:20,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:20,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:20,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:20,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/8ddb602df90f4d0d88f7d6d0ef41210c is 50, key is test_row_0/B:col10/1732481359688/Put/seqid=0 2024-11-24T20:49:20,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T20:49:20,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742401_1577 (size=12151) 2024-11-24T20:49:20,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/8ddb602df90f4d0d88f7d6d0ef41210c 2024-11-24T20:49:20,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/877c51204f814249a0c5a1628b1e2624 is 50, key is test_row_0/C:col10/1732481359688/Put/seqid=0 2024-11-24T20:49:20,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742402_1578 (size=12151) 2024-11-24T20:49:20,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/877c51204f814249a0c5a1628b1e2624 2024-11-24T20:49:20,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/7307b0dca80044fc8f30fac410bab605 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/7307b0dca80044fc8f30fac410bab605 2024-11-24T20:49:20,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/7307b0dca80044fc8f30fac410bab605, entries=150, sequenceid=174, filesize=30.4 K 2024-11-24T20:49:20,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/8ddb602df90f4d0d88f7d6d0ef41210c as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/8ddb602df90f4d0d88f7d6d0ef41210c 2024-11-24T20:49:20,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/8ddb602df90f4d0d88f7d6d0ef41210c, entries=150, sequenceid=174, filesize=11.9 K 2024-11-24T20:49:20,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/877c51204f814249a0c5a1628b1e2624 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/877c51204f814249a0c5a1628b1e2624 2024-11-24T20:49:20,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/877c51204f814249a0c5a1628b1e2624, entries=150, sequenceid=174, filesize=11.9 K 2024-11-24T20:49:20,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 5eaa0424086a20a4fd6f784698d02aeb in 517ms, sequenceid=174, compaction requested=true 2024-11-24T20:49:20,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:20,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaa0424086a20a4fd6f784698d02aeb:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:49:20,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:49:20,205 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:49:20,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaa0424086a20a4fd6f784698d02aeb:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:49:20,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:49:20,206 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:49:20,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaa0424086a20a4fd6f784698d02aeb:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:49:20,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:49:20,206 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:49:20,206 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93539 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:49:20,206 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 5eaa0424086a20a4fd6f784698d02aeb/B is initiating minor compaction (all files) 2024-11-24T20:49:20,206 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 5eaa0424086a20a4fd6f784698d02aeb/A is initiating minor compaction (all files) 2024-11-24T20:49:20,206 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaa0424086a20a4fd6f784698d02aeb/B in TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:20,206 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaa0424086a20a4fd6f784698d02aeb/A in TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:20,206 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/5da84091243c472bbb9940c99aa6004e, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/df5a65b30dea46209d1da9589a8f05a7, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/8ddb602df90f4d0d88f7d6d0ef41210c] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp, totalSize=35.8 K 2024-11-24T20:49:20,206 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/b93342339047408e82f7fb869aa435f4, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/22968dc3c8734110b69e52867e015885, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/7307b0dca80044fc8f30fac410bab605] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp, totalSize=91.3 K 2024-11-24T20:49:20,207 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:20,207 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. files: [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/b93342339047408e82f7fb869aa435f4, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/22968dc3c8734110b69e52867e015885, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/7307b0dca80044fc8f30fac410bab605] 2024-11-24T20:49:20,207 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting b93342339047408e82f7fb869aa435f4, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732481357238 2024-11-24T20:49:20,207 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 5da84091243c472bbb9940c99aa6004e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732481357238 2024-11-24T20:49:20,207 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22968dc3c8734110b69e52867e015885, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732481358404 2024-11-24T20:49:20,207 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting df5a65b30dea46209d1da9589a8f05a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732481358404 2024-11-24T20:49:20,207 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ddb602df90f4d0d88f7d6d0ef41210c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732481359051 2024-11-24T20:49:20,207 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7307b0dca80044fc8f30fac410bab605, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732481359051 2024-11-24T20:49:20,213 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:20,214 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaa0424086a20a4fd6f784698d02aeb#B#compaction#494 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:49:20,215 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/d7ea77bd9eb04f4fae87e673e7756fdd is 50, key is test_row_0/B:col10/1732481359688/Put/seqid=0 2024-11-24T20:49:20,215 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124f4588061217c4f21bdd41b81c7351f9f_5eaa0424086a20a4fd6f784698d02aeb store=[table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:20,217 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124f4588061217c4f21bdd41b81c7351f9f_5eaa0424086a20a4fd6f784698d02aeb, store=[table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:20,217 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f4588061217c4f21bdd41b81c7351f9f_5eaa0424086a20a4fd6f784698d02aeb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:20,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742403_1579 (size=12527) 2024-11-24T20:49:20,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742404_1580 (size=4469) 2024-11-24T20:49:20,239 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaa0424086a20a4fd6f784698d02aeb#A#compaction#495 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:49:20,239 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/5edf133c85a94b13ba21cb51189276f7 is 175, key is test_row_0/A:col10/1732481359688/Put/seqid=0 2024-11-24T20:49:20,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742405_1581 (size=31481) 2024-11-24T20:49:20,253 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/5edf133c85a94b13ba21cb51189276f7 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/5edf133c85a94b13ba21cb51189276f7 2024-11-24T20:49:20,257 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eaa0424086a20a4fd6f784698d02aeb/A of 5eaa0424086a20a4fd6f784698d02aeb into 5edf133c85a94b13ba21cb51189276f7(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:49:20,257 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:20,257 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., storeName=5eaa0424086a20a4fd6f784698d02aeb/A, priority=13, startTime=1732481360205; duration=0sec 2024-11-24T20:49:20,257 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:49:20,257 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaa0424086a20a4fd6f784698d02aeb:A 2024-11-24T20:49:20,258 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:49:20,258 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:49:20,258 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 5eaa0424086a20a4fd6f784698d02aeb/C is initiating minor compaction (all files) 2024-11-24T20:49:20,258 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaa0424086a20a4fd6f784698d02aeb/C in TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:20,259 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/ddb44ad9d34d4d6e9eaecf002cb98c7f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/63e74b1d5cdb4e4c9a82ead7d3abf047, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/877c51204f814249a0c5a1628b1e2624] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp, totalSize=35.8 K 2024-11-24T20:49:20,259 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddb44ad9d34d4d6e9eaecf002cb98c7f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732481357238 2024-11-24T20:49:20,259 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63e74b1d5cdb4e4c9a82ead7d3abf047, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732481358404 2024-11-24T20:49:20,259 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 877c51204f814249a0c5a1628b1e2624, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732481359051 2024-11-24T20:49:20,265 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaa0424086a20a4fd6f784698d02aeb#C#compaction#496 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:49:20,266 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/09656784944f48d0bcf6890ffdfe2b7a is 50, key is test_row_0/C:col10/1732481359688/Put/seqid=0 2024-11-24T20:49:20,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742406_1582 (size=12527) 2024-11-24T20:49:20,281 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/09656784944f48d0bcf6890ffdfe2b7a as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/09656784944f48d0bcf6890ffdfe2b7a 2024-11-24T20:49:20,287 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eaa0424086a20a4fd6f784698d02aeb/C of 5eaa0424086a20a4fd6f784698d02aeb into 09656784944f48d0bcf6890ffdfe2b7a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:49:20,287 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:20,287 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., storeName=5eaa0424086a20a4fd6f784698d02aeb/C, priority=13, startTime=1732481360206; duration=0sec 2024-11-24T20:49:20,287 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:49:20,287 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaa0424086a20a4fd6f784698d02aeb:C 2024-11-24T20:49:20,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-24T20:49:20,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:20,297 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 5eaa0424086a20a4fd6f784698d02aeb 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-24T20:49:20,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243e7f5eacc7ba46c0a77513d6545a585d_5eaa0424086a20a4fd6f784698d02aeb is 50, key is test_row_0/A:col10/1732481359721/Put/seqid=0 2024-11-24T20:49:20,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742407_1583 (size=12304) 2024-11-24T20:49:20,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:49:20,319 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243e7f5eacc7ba46c0a77513d6545a585d_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243e7f5eacc7ba46c0a77513d6545a585d_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:20,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/45c7009754f04f58a4424acf1781d6d2, store: [table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:20,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/45c7009754f04f58a4424acf1781d6d2 is 175, key is test_row_0/A:col10/1732481359721/Put/seqid=0 2024-11-24T20:49:20,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742408_1584 (size=31105) 2024-11-24T20:49:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:20,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:20,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481420363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481420364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:20,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:20,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481420365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:20,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:20,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:20,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:20,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481420366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:20,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:20,369 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481420366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:20,369 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,369 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:20,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:20,444 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T20:49:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T20:49:20,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481420475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:20,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481420475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:20,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481420476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481420476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:20,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:20,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481420476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:20,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:20,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:20,480 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,480 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,480 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:20,630 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/d7ea77bd9eb04f4fae87e673e7756fdd as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/d7ea77bd9eb04f4fae87e673e7756fdd 2024-11-24T20:49:20,637 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eaa0424086a20a4fd6f784698d02aeb/B of 5eaa0424086a20a4fd6f784698d02aeb into d7ea77bd9eb04f4fae87e673e7756fdd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:49:20,637 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:20,637 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., storeName=5eaa0424086a20a4fd6f784698d02aeb/B, priority=13, startTime=1732481360205; duration=0sec 2024-11-24T20:49:20,637 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:49:20,637 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaa0424086a20a4fd6f784698d02aeb:B 2024-11-24T20:49:20,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:20,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481420685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:20,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:20,687 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,687 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481420685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:20,687 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:20,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481420686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:20,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481420686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:20,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481420686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:20,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:20,731 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/45c7009754f04f58a4424acf1781d6d2 2024-11-24T20:49:20,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/c16c6b249f31432eb4796a3b36387ae9 is 50, key is test_row_0/B:col10/1732481359721/Put/seqid=0 2024-11-24T20:49:20,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742409_1585 (size=12151) 2024-11-24T20:49:20,749 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/c16c6b249f31432eb4796a3b36387ae9 2024-11-24T20:49:20,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/311f582fd2e343d3a0135d1c7d411b22 is 50, key is test_row_0/C:col10/1732481359721/Put/seqid=0 2024-11-24T20:49:20,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742410_1586 (size=12151) 2024-11-24T20:49:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T20:49:20,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:20,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481420996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:20,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:20,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481420996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:20,998 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,998 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:20,998 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:20,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:20,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481420996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:20,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:20,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:20,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481420997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:20,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:20,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481420997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:20,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:20,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:20,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:21,161 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/311f582fd2e343d3a0135d1c7d411b22 2024-11-24T20:49:21,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/45c7009754f04f58a4424acf1781d6d2 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/45c7009754f04f58a4424acf1781d6d2 2024-11-24T20:49:21,167 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/45c7009754f04f58a4424acf1781d6d2, entries=150, sequenceid=198, filesize=30.4 K 2024-11-24T20:49:21,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/c16c6b249f31432eb4796a3b36387ae9 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/c16c6b249f31432eb4796a3b36387ae9 2024-11-24T20:49:21,170 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/c16c6b249f31432eb4796a3b36387ae9, entries=150, sequenceid=198, filesize=11.9 K 2024-11-24T20:49:21,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/311f582fd2e343d3a0135d1c7d411b22 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/311f582fd2e343d3a0135d1c7d411b22 2024-11-24T20:49:21,174 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/311f582fd2e343d3a0135d1c7d411b22, entries=150, sequenceid=198, filesize=11.9 K 2024-11-24T20:49:21,174 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 5eaa0424086a20a4fd6f784698d02aeb in 877ms, sequenceid=198, compaction requested=false 2024-11-24T20:49:21,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:21,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:21,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-24T20:49:21,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-24T20:49:21,177 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-24T20:49:21,178 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3390 sec 2024-11-24T20:49:21,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.3430 sec 2024-11-24T20:49:21,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:21,507 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:49:21,507 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5eaa0424086a20a4fd6f784698d02aeb 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-24T20:49:21,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411247afb4bbe6bfc43cbb57c726b9e03099b_5eaa0424086a20a4fd6f784698d02aeb is 50, key is test_row_0/A:col10/1732481361506/Put/seqid=0 2024-11-24T20:49:21,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742411_1587 (size=12304) 2024-11-24T20:49:21,530 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:49:21,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:21,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481421522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:21,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:21,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:21,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:21,535 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411247afb4bbe6bfc43cbb57c726b9e03099b_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247afb4bbe6bfc43cbb57c726b9e03099b_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:21,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:21,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:21,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481421531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:21,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481421531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:21,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:21,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:21,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481421531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:21,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481421532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:21,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:21,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:21,538 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:21,540 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/4f8ae702c1a24cd6985a38e5d6b243b3, store: [table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:21,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/4f8ae702c1a24cd6985a38e5d6b243b3 is 175, key is test_row_0/A:col10/1732481361506/Put/seqid=0 2024-11-24T20:49:21,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742412_1588 (size=31105) 2024-11-24T20:49:21,566 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/4f8ae702c1a24cd6985a38e5d6b243b3 2024-11-24T20:49:21,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/0916d1c4e0cc450c9ce911ef4e6c9b5c is 50, key is test_row_0/B:col10/1732481361506/Put/seqid=0 2024-11-24T20:49:21,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742413_1589 (size=12151) 2024-11-24T20:49:21,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:21,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481421635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:21,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:21,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:21,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481421645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:21,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:21,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481421645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:21,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:21,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481421645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:21,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:21,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:21,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:21,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:21,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481421645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:21,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:21,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:21,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:21,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:21,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481421845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:21,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:21,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:21,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:21,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:21,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481421855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:21,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:21,859 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:21,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481421855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:21,859 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:21,859 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:21,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:21,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:21,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481421855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:21,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:21,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:21,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:21,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481421856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:21,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:21,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:21,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:21,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:21,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:21,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T20:49:21,991 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:49:21,992 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:49:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-24T20:49:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T20:49:21,994 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:49:21,995 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:49:21,995 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:49:22,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/0916d1c4e0cc450c9ce911ef4e6c9b5c 2024-11-24T20:49:22,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/61fa0c29f0c8462fb845c84c4d4e1bc8 is 50, key is test_row_0/C:col10/1732481361506/Put/seqid=0 2024-11-24T20:49:22,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742414_1590 (size=12151) 2024-11-24T20:49:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T20:49:22,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T20:49:22,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:22,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:22,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:22,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:22,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:22,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:22,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481422155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:22,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:22,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:22,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:22,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481422165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:22,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:22,167 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481422165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:22,167 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:22,167 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:22,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:22,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:22,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481422165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:22,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:22,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:22,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:22,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:22,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481422165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:22,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:22,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:22,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:22,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T20:49:22,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:22,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:22,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:22,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:22,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T20:49:22,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/61fa0c29f0c8462fb845c84c4d4e1bc8 2024-11-24T20:49:22,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/4f8ae702c1a24cd6985a38e5d6b243b3 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/4f8ae702c1a24cd6985a38e5d6b243b3 2024-11-24T20:49:22,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/4f8ae702c1a24cd6985a38e5d6b243b3, entries=150, sequenceid=216, filesize=30.4 K 2024-11-24T20:49:22,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/0916d1c4e0cc450c9ce911ef4e6c9b5c as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/0916d1c4e0cc450c9ce911ef4e6c9b5c 2024-11-24T20:49:22,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/0916d1c4e0cc450c9ce911ef4e6c9b5c, entries=150, sequenceid=216, filesize=11.9 K 2024-11-24T20:49:22,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/61fa0c29f0c8462fb845c84c4d4e1bc8 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/61fa0c29f0c8462fb845c84c4d4e1bc8 2024-11-24T20:49:22,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/61fa0c29f0c8462fb845c84c4d4e1bc8, entries=150, sequenceid=216, filesize=11.9 K 2024-11-24T20:49:22,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 5eaa0424086a20a4fd6f784698d02aeb in 932ms, sequenceid=216, compaction requested=true 2024-11-24T20:49:22,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:22,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaa0424086a20a4fd6f784698d02aeb:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:49:22,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:49:22,440 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:49:22,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaa0424086a20a4fd6f784698d02aeb:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:49:22,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:49:22,440 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:49:22,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaa0424086a20a4fd6f784698d02aeb:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:49:22,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:49:22,440 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:49:22,440 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93691 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:49:22,440 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 5eaa0424086a20a4fd6f784698d02aeb/A is initiating minor compaction (all files) 2024-11-24T20:49:22,440 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 5eaa0424086a20a4fd6f784698d02aeb/B is initiating minor compaction (all files) 2024-11-24T20:49:22,440 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaa0424086a20a4fd6f784698d02aeb/A in TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:22,440 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaa0424086a20a4fd6f784698d02aeb/B in TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:22,441 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/5edf133c85a94b13ba21cb51189276f7, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/45c7009754f04f58a4424acf1781d6d2, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/4f8ae702c1a24cd6985a38e5d6b243b3] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp, totalSize=91.5 K 2024-11-24T20:49:22,441 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/d7ea77bd9eb04f4fae87e673e7756fdd, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/c16c6b249f31432eb4796a3b36387ae9, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/0916d1c4e0cc450c9ce911ef4e6c9b5c] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp, totalSize=36.0 K 2024-11-24T20:49:22,441 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:22,441 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. files: [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/5edf133c85a94b13ba21cb51189276f7, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/45c7009754f04f58a4424acf1781d6d2, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/4f8ae702c1a24cd6985a38e5d6b243b3] 2024-11-24T20:49:22,441 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting d7ea77bd9eb04f4fae87e673e7756fdd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732481359051 2024-11-24T20:49:22,441 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5edf133c85a94b13ba21cb51189276f7, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732481359051 2024-11-24T20:49:22,441 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting c16c6b249f31432eb4796a3b36387ae9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732481359712 2024-11-24T20:49:22,441 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45c7009754f04f58a4424acf1781d6d2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732481359712 2024-11-24T20:49:22,441 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 0916d1c4e0cc450c9ce911ef4e6c9b5c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732481360364 2024-11-24T20:49:22,441 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f8ae702c1a24cd6985a38e5d6b243b3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732481360364 2024-11-24T20:49:22,445 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:22,446 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaa0424086a20a4fd6f784698d02aeb#B#compaction#503 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:49:22,447 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/527940924315465e8d03ef76f272541d is 50, key is test_row_0/B:col10/1732481361506/Put/seqid=0 2024-11-24T20:49:22,452 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411249cc1d585d15e4edf886a7c30b33bddec_5eaa0424086a20a4fd6f784698d02aeb store=[table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:22,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T20:49:22,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:22,453 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 5eaa0424086a20a4fd6f784698d02aeb 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-24T20:49:22,454 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411249cc1d585d15e4edf886a7c30b33bddec_5eaa0424086a20a4fd6f784698d02aeb, store=[table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:22,454 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411249cc1d585d15e4edf886a7c30b33bddec_5eaa0424086a20a4fd6f784698d02aeb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:22,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742415_1591 (size=12629) 2024-11-24T20:49:22,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742416_1592 (size=4469) 2024-11-24T20:49:22,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f291050713cc444f9ec48f6ea9bbacea_5eaa0424086a20a4fd6f784698d02aeb is 50, key is test_row_0/A:col10/1732481361530/Put/seqid=0 2024-11-24T20:49:22,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742417_1593 (size=12304) 2024-11-24T20:49:22,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:49:22,466 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f291050713cc444f9ec48f6ea9bbacea_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f291050713cc444f9ec48f6ea9bbacea_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:22,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/853a03f53b834f9db46b5a1c4b00068a, store: [table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:22,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/853a03f53b834f9db46b5a1c4b00068a is 175, key is test_row_0/A:col10/1732481361530/Put/seqid=0 2024-11-24T20:49:22,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742418_1594 (size=31105) 2024-11-24T20:49:22,471 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=236, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/853a03f53b834f9db46b5a1c4b00068a 2024-11-24T20:49:22,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/a03a2e0299ad411bb354574c438bab49 is 50, key is test_row_0/B:col10/1732481361530/Put/seqid=0 2024-11-24T20:49:22,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742419_1595 (size=12151) 2024-11-24T20:49:22,481 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/a03a2e0299ad411bb354574c438bab49 2024-11-24T20:49:22,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/c8864fc6f805484ebda6507309de054d is 50, key is test_row_0/C:col10/1732481361530/Put/seqid=0 2024-11-24T20:49:22,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742420_1596 (size=12151) 2024-11-24T20:49:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T20:49:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:22,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:22,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481422684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:22,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:22,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:22,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:22,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481422685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:22,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:22,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481422689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:22,695 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:22,695 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:22,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:22,695 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481422689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:22,695 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:22,695 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:22,695 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:22,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481422691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:22,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:22,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:22,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:22,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481422795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:22,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:22,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:22,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:22,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481422795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:22,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:22,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:22,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:22,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481422805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:22,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:22,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:22,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:22,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481422805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:22,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481422805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:22,809 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:22,809 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:22,809 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:22,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:22,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:22,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:22,859 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaa0424086a20a4fd6f784698d02aeb#A#compaction#504 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:49:22,859 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/662c8328953a4772901dcf404d93e895 is 175, key is test_row_0/A:col10/1732481361506/Put/seqid=0 2024-11-24T20:49:22,862 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/527940924315465e8d03ef76f272541d as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/527940924315465e8d03ef76f272541d 2024-11-24T20:49:22,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742421_1597 (size=31583) 2024-11-24T20:49:22,870 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eaa0424086a20a4fd6f784698d02aeb/B of 5eaa0424086a20a4fd6f784698d02aeb into 527940924315465e8d03ef76f272541d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:49:22,870 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:22,870 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., storeName=5eaa0424086a20a4fd6f784698d02aeb/B, priority=13, startTime=1732481362440; duration=0sec 2024-11-24T20:49:22,871 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:49:22,871 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaa0424086a20a4fd6f784698d02aeb:B 2024-11-24T20:49:22,871 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:49:22,872 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:49:22,872 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 5eaa0424086a20a4fd6f784698d02aeb/C is initiating minor compaction (all files) 2024-11-24T20:49:22,872 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaa0424086a20a4fd6f784698d02aeb/C in TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:22,872 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/09656784944f48d0bcf6890ffdfe2b7a, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/311f582fd2e343d3a0135d1c7d411b22, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/61fa0c29f0c8462fb845c84c4d4e1bc8] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp, totalSize=36.0 K 2024-11-24T20:49:22,872 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 09656784944f48d0bcf6890ffdfe2b7a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732481359051 2024-11-24T20:49:22,873 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 311f582fd2e343d3a0135d1c7d411b22, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732481359712 2024-11-24T20:49:22,873 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 61fa0c29f0c8462fb845c84c4d4e1bc8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732481360364 2024-11-24T20:49:22,880 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaa0424086a20a4fd6f784698d02aeb#C#compaction#508 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:49:22,880 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/a61f292c19b9493bb737526258e9b3d9 is 50, key is test_row_0/C:col10/1732481361506/Put/seqid=0 2024-11-24T20:49:22,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742422_1598 (size=12629) 2024-11-24T20:49:22,895 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/c8864fc6f805484ebda6507309de054d 2024-11-24T20:49:22,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/853a03f53b834f9db46b5a1c4b00068a as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/853a03f53b834f9db46b5a1c4b00068a 2024-11-24T20:49:22,909 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/853a03f53b834f9db46b5a1c4b00068a, entries=150, sequenceid=236, filesize=30.4 K 2024-11-24T20:49:22,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/a03a2e0299ad411bb354574c438bab49 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/a03a2e0299ad411bb354574c438bab49 2024-11-24T20:49:22,914 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/a03a2e0299ad411bb354574c438bab49, entries=150, sequenceid=236, filesize=11.9 K 2024-11-24T20:49:22,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/c8864fc6f805484ebda6507309de054d as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/c8864fc6f805484ebda6507309de054d 2024-11-24T20:49:22,919 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/c8864fc6f805484ebda6507309de054d, entries=150, sequenceid=236, filesize=11.9 K 2024-11-24T20:49:22,920 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 5eaa0424086a20a4fd6f784698d02aeb in 467ms, sequenceid=236, compaction requested=false 2024-11-24T20:49:22,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:22,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:22,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-24T20:49:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-24T20:49:22,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-24T20:49:22,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 927 msec 2024-11-24T20:49:22,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 933 msec 2024-11-24T20:49:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:23,009 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:49:23,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5eaa0424086a20a4fd6f784698d02aeb 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-24T20:49:23,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124891e4389b7f94a73bb2f17b01bd21951_5eaa0424086a20a4fd6f784698d02aeb is 50, key is test_row_0/A:col10/1732481363007/Put/seqid=0 2024-11-24T20:49:23,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742423_1599 (size=14794) 2024-11-24T20:49:23,030 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:49:23,033 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124891e4389b7f94a73bb2f17b01bd21951_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124891e4389b7f94a73bb2f17b01bd21951_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:23,034 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/f0f0fe87494544e19befac2ae61602f6, store: [table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:23,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/f0f0fe87494544e19befac2ae61602f6 is 175, key is test_row_0/A:col10/1732481363007/Put/seqid=0 2024-11-24T20:49:23,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481423033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:23,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,039 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:23,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481423037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:23,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481423038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:23,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481423039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:23,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481423039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:23,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742424_1600 (size=39749) 2024-11-24T20:49:23,058 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/f0f0fe87494544e19befac2ae61602f6 2024-11-24T20:49:23,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/a65c73ba7aa24396a0d5c9266b594b47 is 50, key is test_row_0/B:col10/1732481363007/Put/seqid=0 2024-11-24T20:49:23,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742425_1601 (size=12151) 2024-11-24T20:49:23,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/a65c73ba7aa24396a0d5c9266b594b47 2024-11-24T20:49:23,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/e0da9490eb15404b983e13a19e62986c is 50, key is test_row_0/C:col10/1732481363007/Put/seqid=0 2024-11-24T20:49:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T20:49:23,135 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:49:23,136 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:49:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-24T20:49:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T20:49:23,138 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:49:23,138 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:49:23,139 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:49:23,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742426_1602 (size=12151) 2024-11-24T20:49:23,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/e0da9490eb15404b983e13a19e62986c 2024-11-24T20:49:23,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:23,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481423145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:23,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:23,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481423155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:23,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:23,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481423155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:23,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:23,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,158 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,158 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:23,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/f0f0fe87494544e19befac2ae61602f6 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/f0f0fe87494544e19befac2ae61602f6 2024-11-24T20:49:23,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:23,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481423156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:23,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:23,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481423156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:23,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:23,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:23,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/f0f0fe87494544e19befac2ae61602f6, entries=200, sequenceid=254, filesize=38.8 K 2024-11-24T20:49:23,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/a65c73ba7aa24396a0d5c9266b594b47 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/a65c73ba7aa24396a0d5c9266b594b47 2024-11-24T20:49:23,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/a65c73ba7aa24396a0d5c9266b594b47, entries=150, sequenceid=254, filesize=11.9 K 2024-11-24T20:49:23,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/e0da9490eb15404b983e13a19e62986c as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/e0da9490eb15404b983e13a19e62986c 2024-11-24T20:49:23,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/e0da9490eb15404b983e13a19e62986c, entries=150, sequenceid=254, filesize=11.9 K 2024-11-24T20:49:23,179 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 5eaa0424086a20a4fd6f784698d02aeb in 169ms, sequenceid=254, compaction requested=true 2024-11-24T20:49:23,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:23,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaa0424086a20a4fd6f784698d02aeb:A, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:49:23,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:49:23,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaa0424086a20a4fd6f784698d02aeb:B, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:49:23,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T20:49:23,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaa0424086a20a4fd6f784698d02aeb:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:49:23,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-24T20:49:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T20:49:23,272 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/662c8328953a4772901dcf404d93e895 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/662c8328953a4772901dcf404d93e895 2024-11-24T20:49:23,277 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eaa0424086a20a4fd6f784698d02aeb/A of 5eaa0424086a20a4fd6f784698d02aeb into 662c8328953a4772901dcf404d93e895(size=30.8 K), total size for store is 100.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:49:23,277 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:23,277 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., storeName=5eaa0424086a20a4fd6f784698d02aeb/A, priority=13, startTime=1732481362440; duration=0sec 2024-11-24T20:49:23,277 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-24T20:49:23,277 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaa0424086a20a4fd6f784698d02aeb:A 2024-11-24T20:49:23,277 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaa0424086a20a4fd6f784698d02aeb:A 2024-11-24T20:49:23,278 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-11-24T20:49:23,279 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-24T20:49:23,279 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-24T20:49:23,279 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. because compaction request was cancelled 2024-11-24T20:49:23,279 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaa0424086a20a4fd6f784698d02aeb:C 2024-11-24T20:49:23,279 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:49:23,280 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102437 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:49:23,280 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 5eaa0424086a20a4fd6f784698d02aeb/A is initiating minor compaction (all files) 2024-11-24T20:49:23,280 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaa0424086a20a4fd6f784698d02aeb/A in TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:23,280 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/662c8328953a4772901dcf404d93e895, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/853a03f53b834f9db46b5a1c4b00068a, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/f0f0fe87494544e19befac2ae61602f6] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp, totalSize=100.0 K 2024-11-24T20:49:23,280 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:23,280 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. files: [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/662c8328953a4772901dcf404d93e895, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/853a03f53b834f9db46b5a1c4b00068a, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/f0f0fe87494544e19befac2ae61602f6] 2024-11-24T20:49:23,280 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 662c8328953a4772901dcf404d93e895, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732481360364 2024-11-24T20:49:23,281 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 853a03f53b834f9db46b5a1c4b00068a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732481361528 2024-11-24T20:49:23,281 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0f0fe87494544e19befac2ae61602f6, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732481362678 2024-11-24T20:49:23,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-24T20:49:23,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:23,291 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 5eaa0424086a20a4fd6f784698d02aeb 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-24T20:49:23,294 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/a61f292c19b9493bb737526258e9b3d9 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/a61f292c19b9493bb737526258e9b3d9 2024-11-24T20:49:23,300 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eaa0424086a20a4fd6f784698d02aeb/C of 5eaa0424086a20a4fd6f784698d02aeb into a61f292c19b9493bb737526258e9b3d9(size=12.3 K), total size for store is 36.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:49:23,300 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:23,300 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., storeName=5eaa0424086a20a4fd6f784698d02aeb/C, priority=13, startTime=1732481362440; duration=0sec 2024-11-24T20:49:23,300 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:49:23,300 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaa0424086a20a4fd6f784698d02aeb:C 2024-11-24T20:49:23,300 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:49:23,300 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:23,301 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:49:23,301 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 5eaa0424086a20a4fd6f784698d02aeb/B is initiating minor compaction (all files) 2024-11-24T20:49:23,301 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaa0424086a20a4fd6f784698d02aeb/B in TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:23,302 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/527940924315465e8d03ef76f272541d, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/a03a2e0299ad411bb354574c438bab49, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/a65c73ba7aa24396a0d5c9266b594b47] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp, totalSize=36.1 K 2024-11-24T20:49:23,303 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 527940924315465e8d03ef76f272541d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732481360364 2024-11-24T20:49:23,303 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting a03a2e0299ad411bb354574c438bab49, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732481361528 2024-11-24T20:49:23,303 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting a65c73ba7aa24396a0d5c9266b594b47, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732481362678 2024-11-24T20:49:23,305 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112425a3ace3a3ee4c0abdc6dbbb5478552d_5eaa0424086a20a4fd6f784698d02aeb store=[table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:23,307 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112425a3ace3a3ee4c0abdc6dbbb5478552d_5eaa0424086a20a4fd6f784698d02aeb, store=[table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:23,308 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112425a3ace3a3ee4c0abdc6dbbb5478552d_5eaa0424086a20a4fd6f784698d02aeb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:23,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b62b6a85f5f34f8eacec5c9d1daca17b_5eaa0424086a20a4fd6f784698d02aeb is 50, key is test_row_0/A:col10/1732481363037/Put/seqid=0 2024-11-24T20:49:23,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742427_1603 (size=4469) 2024-11-24T20:49:23,313 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaa0424086a20a4fd6f784698d02aeb#B#compaction#514 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:49:23,314 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/9fe001aa38064fbf92b6776ca504ad5c is 50, key is test_row_0/B:col10/1732481363007/Put/seqid=0 2024-11-24T20:49:23,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742428_1604 (size=12454) 2024-11-24T20:49:23,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742429_1605 (size=12731) 2024-11-24T20:49:23,345 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/9fe001aa38064fbf92b6776ca504ad5c as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/9fe001aa38064fbf92b6776ca504ad5c 2024-11-24T20:49:23,349 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eaa0424086a20a4fd6f784698d02aeb/B of 5eaa0424086a20a4fd6f784698d02aeb into 9fe001aa38064fbf92b6776ca504ad5c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:49:23,349 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:23,349 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., storeName=5eaa0424086a20a4fd6f784698d02aeb/B, priority=13, startTime=1732481363179; duration=0sec 2024-11-24T20:49:23,349 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:49:23,349 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaa0424086a20a4fd6f784698d02aeb:B 2024-11-24T20:49:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:23,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:23,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:23,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481423378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,385 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:23,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481423379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:23,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:23,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481423385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:23,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:23,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481423385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:23,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:23,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:23,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:23,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481423385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:23,392 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,392 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,392 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:23,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T20:49:23,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481423495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481423495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481423495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:23,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481423495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481423495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,499 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:23,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:23,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481423705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:23,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:23,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481423705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481423705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:23,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481423705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:23,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:23,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481423705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:23,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:23,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:23,712 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaa0424086a20a4fd6f784698d02aeb#A#compaction#512 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:49:23,713 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/97f985660c054afd9761eda24434460d is 175, key is test_row_0/A:col10/1732481363007/Put/seqid=0 2024-11-24T20:49:23,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742430_1606 (size=31685) 2024-11-24T20:49:23,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:49:23,729 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b62b6a85f5f34f8eacec5c9d1daca17b_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b62b6a85f5f34f8eacec5c9d1daca17b_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:23,729 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/97f985660c054afd9761eda24434460d as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/97f985660c054afd9761eda24434460d 2024-11-24T20:49:23,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/6e68ce05ee3b43cb9fca6c59a5028835, store: [table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:23,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/6e68ce05ee3b43cb9fca6c59a5028835 is 175, key is test_row_0/A:col10/1732481363037/Put/seqid=0 2024-11-24T20:49:23,734 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eaa0424086a20a4fd6f784698d02aeb/A of 5eaa0424086a20a4fd6f784698d02aeb into 97f985660c054afd9761eda24434460d(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:49:23,734 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:23,734 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., storeName=5eaa0424086a20a4fd6f784698d02aeb/A, priority=13, startTime=1732481363179; duration=0sec 2024-11-24T20:49:23,734 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:49:23,734 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaa0424086a20a4fd6f784698d02aeb:A 2024-11-24T20:49:23,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742431_1607 (size=31255) 2024-11-24T20:49:23,737 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/6e68ce05ee3b43cb9fca6c59a5028835 2024-11-24T20:49:23,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/307b323a078546afa2753b764e1b784b is 50, key is test_row_0/B:col10/1732481363037/Put/seqid=0 2024-11-24T20:49:23,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742432_1608 (size=12301) 2024-11-24T20:49:23,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T20:49:24,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481424015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:24,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:24,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481424015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:24,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:24,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:24,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:24,017 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481424015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:24,017 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:24,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:24,017 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481424015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:24,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:24,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:24,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481424015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:24,018 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:24,158 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/307b323a078546afa2753b764e1b784b 2024-11-24T20:49:24,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/66a0efed06434c1aac3ee9729979b17f is 50, key is test_row_0/C:col10/1732481363037/Put/seqid=0 2024-11-24T20:49:24,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742433_1609 (size=12301) 2024-11-24T20:49:24,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T20:49:24,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:24,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481424525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:24,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:24,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:24,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481424525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:24,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:24,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:24,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:24,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481424525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:24,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:24,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:24,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:24,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:24,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:24,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:24,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:24,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481424525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:24,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:24,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:24,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:24,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:24,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481424525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:24,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:24,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:24,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:24,571 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/66a0efed06434c1aac3ee9729979b17f 2024-11-24T20:49:24,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/6e68ce05ee3b43cb9fca6c59a5028835 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/6e68ce05ee3b43cb9fca6c59a5028835 2024-11-24T20:49:24,582 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/6e68ce05ee3b43cb9fca6c59a5028835, entries=150, sequenceid=275, filesize=30.5 K 2024-11-24T20:49:24,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/307b323a078546afa2753b764e1b784b as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/307b323a078546afa2753b764e1b784b 2024-11-24T20:49:24,585 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/307b323a078546afa2753b764e1b784b, entries=150, sequenceid=275, filesize=12.0 K 2024-11-24T20:49:24,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/66a0efed06434c1aac3ee9729979b17f as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/66a0efed06434c1aac3ee9729979b17f 2024-11-24T20:49:24,588 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/66a0efed06434c1aac3ee9729979b17f, entries=150, sequenceid=275, filesize=12.0 K 2024-11-24T20:49:24,589 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 5eaa0424086a20a4fd6f784698d02aeb in 1298ms, sequenceid=275, compaction requested=true 2024-11-24T20:49:24,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:24,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:24,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-24T20:49:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-24T20:49:24,592 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-24T20:49:24,592 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4520 sec 2024-11-24T20:49:24,593 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.4560 sec 2024-11-24T20:49:25,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T20:49:25,295 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:49:25,296 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:49:25,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-24T20:49:25,300 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:49:25,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-24T20:49:25,302 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:49:25,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:49:25,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-24T20:49:25,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-24T20:49:25,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:25,456 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 5eaa0424086a20a4fd6f784698d02aeb 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-24T20:49:25,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c23b3a040d024fbbae840c4093471080_5eaa0424086a20a4fd6f784698d02aeb is 50, key is test_row_0/A:col10/1732481363373/Put/seqid=0 2024-11-24T20:49:25,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742434_1610 (size=12454) 2024-11-24T20:49:25,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:25,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:25,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:25,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481425556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:25,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:25,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481425557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:25,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:25,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:25,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:25,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:25,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481425558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:25,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:25,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:25,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:25,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481425559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:25,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:25,564 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:25,564 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:25,564 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:25,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:25,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481425561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:25,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:25,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:25,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:25,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-24T20:49:25,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:25,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481425665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:25,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:25,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:25,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481425665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:25,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:25,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:25,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:25,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:25,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:25,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481425675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:25,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481425675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:25,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:25,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:25,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:25,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481425675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:25,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:49:25,872 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c23b3a040d024fbbae840c4093471080_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c23b3a040d024fbbae840c4093471080_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:25,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/6b03291bf0114c6a91dcdaf42b8db7ae, store: [table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:25,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/6b03291bf0114c6a91dcdaf42b8db7ae is 175, key is test_row_0/A:col10/1732481363373/Put/seqid=0 2024-11-24T20:49:25,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:25,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481425875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:25,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:25,878 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:25,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481425875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:25,878 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:25,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742435_1611 (size=31255) 2024-11-24T20:49:25,878 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:25,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:25,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:25,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:25,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:25,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481425885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:25,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:25,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481425885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:25,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:25,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481425885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:25,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-24T20:49:26,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:26,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481426185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:26,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:26,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481426185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:26,189 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:26,189 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:26,189 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:26,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:26,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:26,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:26,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481426196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:26,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481426196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:26,199 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:26,199 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:26,199 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:26,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:26,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:26,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:26,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:26,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481426197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:26,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:26,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:26,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:26,277 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/6b03291bf0114c6a91dcdaf42b8db7ae 2024-11-24T20:49:26,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/e11edd69e60c4509ac97231c71dd04d2 is 50, key is test_row_0/B:col10/1732481363373/Put/seqid=0 2024-11-24T20:49:26,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742436_1612 (size=12301) 2024-11-24T20:49:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-24T20:49:26,686 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/e11edd69e60c4509ac97231c71dd04d2 2024-11-24T20:49:26,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/2a5e580344374f19bea2a2240f26d29e is 50, key is test_row_0/C:col10/1732481363373/Put/seqid=0 2024-11-24T20:49:26,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742437_1613 (size=12301) 2024-11-24T20:49:26,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:26,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481426695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:26,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481426695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:26,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:26,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:26,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:26,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:26,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:26,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:26,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:26,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481426705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:26,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:26,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:26,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:26,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:26,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481426705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:26,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:26,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:26,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:26,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:26,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481426705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:26,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:26,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:26,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:27,097 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/2a5e580344374f19bea2a2240f26d29e 2024-11-24T20:49:27,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/6b03291bf0114c6a91dcdaf42b8db7ae as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/6b03291bf0114c6a91dcdaf42b8db7ae 2024-11-24T20:49:27,103 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/6b03291bf0114c6a91dcdaf42b8db7ae, entries=150, sequenceid=295, filesize=30.5 K 2024-11-24T20:49:27,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/e11edd69e60c4509ac97231c71dd04d2 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/e11edd69e60c4509ac97231c71dd04d2 2024-11-24T20:49:27,106 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/e11edd69e60c4509ac97231c71dd04d2, entries=150, sequenceid=295, filesize=12.0 K 2024-11-24T20:49:27,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/2a5e580344374f19bea2a2240f26d29e as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/2a5e580344374f19bea2a2240f26d29e 2024-11-24T20:49:27,111 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/2a5e580344374f19bea2a2240f26d29e, entries=150, sequenceid=295, filesize=12.0 K 2024-11-24T20:49:27,112 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 5eaa0424086a20a4fd6f784698d02aeb in 1656ms, sequenceid=295, compaction requested=true 2024-11-24T20:49:27,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:27,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:27,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-24T20:49:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-24T20:49:27,115 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-24T20:49:27,115 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8110 sec 2024-11-24T20:49:27,116 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.8190 sec 2024-11-24T20:49:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-24T20:49:27,445 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:49:27,445 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:49:27,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-24T20:49:27,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-24T20:49:27,447 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:49:27,448 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:49:27,448 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:49:27,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-24T20:49:27,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-24T20:49:27,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:27,600 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 5eaa0424086a20a4fd6f784698d02aeb 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-24T20:49:27,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112474fa8fb92a1d4d0ab8993b54bdf3b1a2_5eaa0424086a20a4fd6f784698d02aeb is 50, key is test_row_0/A:col10/1732481365558/Put/seqid=0 2024-11-24T20:49:27,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742438_1614 (size=12454) 2024-11-24T20:49:27,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:49:27,628 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112474fa8fb92a1d4d0ab8993b54bdf3b1a2_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112474fa8fb92a1d4d0ab8993b54bdf3b1a2_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:27,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/6502a2eb5c9948249d908af1335416a0, store: [table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:27,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/6502a2eb5c9948249d908af1335416a0 is 175, key is test_row_0/A:col10/1732481365558/Put/seqid=0 2024-11-24T20:49:27,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742439_1615 (size=31255) 2024-11-24T20:49:27,637 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=314, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/6502a2eb5c9948249d908af1335416a0 2024-11-24T20:49:27,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/b365c601758148b79f4da435b6af1fea is 50, key is test_row_0/B:col10/1732481365558/Put/seqid=0 2024-11-24T20:49:27,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742440_1616 (size=12301) 2024-11-24T20:49:27,656 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/b365c601758148b79f4da435b6af1fea 2024-11-24T20:49:27,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/04b034d0e2f24b4996c1ccd81d47a0c9 is 50, key is test_row_0/C:col10/1732481365558/Put/seqid=0 2024-11-24T20:49:27,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742441_1617 (size=12301) 2024-11-24T20:49:27,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:27,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:27,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481427732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:27,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481427733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:27,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:27,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481427733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:27,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:27,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:27,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481427734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:27,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:27,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:27,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:27,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:27,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:27,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:27,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481427734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:27,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:27,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:27,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:27,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:27,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:27,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:27,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-24T20:49:27,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481427845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:27,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:27,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:27,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:27,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481427846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:27,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:27,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:27,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:27,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481427846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:27,848 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:27,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:27,848 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:27,848 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481427846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:27,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:27,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:27,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:27,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481427846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:27,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:27,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:27,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:28,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:28,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481428055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:28,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:28,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:28,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481428056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:28,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:28,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481428056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:28,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:28,061 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,061 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,061 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:28,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:28,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481428056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:28,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:28,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:28,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481428056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:28,062 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,062 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,062 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-24T20:49:28,086 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/04b034d0e2f24b4996c1ccd81d47a0c9 2024-11-24T20:49:28,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/6502a2eb5c9948249d908af1335416a0 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/6502a2eb5c9948249d908af1335416a0 2024-11-24T20:49:28,099 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/6502a2eb5c9948249d908af1335416a0, entries=150, sequenceid=314, filesize=30.5 K 2024-11-24T20:49:28,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/b365c601758148b79f4da435b6af1fea as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/b365c601758148b79f4da435b6af1fea 2024-11-24T20:49:28,106 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/b365c601758148b79f4da435b6af1fea, entries=150, sequenceid=314, filesize=12.0 K 2024-11-24T20:49:28,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/04b034d0e2f24b4996c1ccd81d47a0c9 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/04b034d0e2f24b4996c1ccd81d47a0c9 2024-11-24T20:49:28,109 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/04b034d0e2f24b4996c1ccd81d47a0c9, entries=150, sequenceid=314, filesize=12.0 K 2024-11-24T20:49:28,110 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 5eaa0424086a20a4fd6f784698d02aeb in 510ms, sequenceid=314, compaction requested=true 2024-11-24T20:49:28,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:28,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:28,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-24T20:49:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-24T20:49:28,115 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-24T20:49:28,117 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 671 msec 2024-11-24T20:49:28,117 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 664 msec 2024-11-24T20:49:28,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:28,368 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:49:28,368 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5eaa0424086a20a4fd6f784698d02aeb 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-24T20:49:28,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240e25f1c210f142e3a071f73342545eac_5eaa0424086a20a4fd6f784698d02aeb is 50, key is test_row_0/A:col10/1732481368367/Put/seqid=0 2024-11-24T20:49:28,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742442_1618 (size=14994) 2024-11-24T20:49:28,382 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:49:28,385 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240e25f1c210f142e3a071f73342545eac_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240e25f1c210f142e3a071f73342545eac_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:28,385 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/13d06b00d9584dc6be8b5c6780b31cb9, store: [table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:28,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/13d06b00d9584dc6be8b5c6780b31cb9 is 175, key is test_row_0/A:col10/1732481368367/Put/seqid=0 2024-11-24T20:49:28,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742443_1619 (size=39949) 2024-11-24T20:49:28,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:28,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:28,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481428382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:28,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481428384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:28,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:28,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481428385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:28,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:28,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:28,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:28,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:28,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481428387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:28,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:28,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:28,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481428387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:28,391 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,391 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,391 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:28,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481428495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:28,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:28,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481428495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:28,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:28,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:28,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481428495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:28,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:28,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:28,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481428496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:28,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:28,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:28,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481428496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:28,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:28,499 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,499 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,499 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:28,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-24T20:49:28,586 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:49:28,586 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:49:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-24T20:49:28,587 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:49:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-24T20:49:28,588 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:49:28,588 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:49:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-24T20:49:28,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481428705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:28,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:28,707 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481428705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:28,707 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,707 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:28,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:28,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481428706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:28,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:28,708 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481428706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:28,708 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,708 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:28,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:28,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481428706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:28,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:28,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:28,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-24T20:49:28,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:28,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:28,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:28,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:28,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:28,791 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=333, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/13d06b00d9584dc6be8b5c6780b31cb9 2024-11-24T20:49:28,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/b4d71f946e34450bb317c3b2754f0098 is 50, key is test_row_0/B:col10/1732481368367/Put/seqid=0 2024-11-24T20:49:28,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742444_1620 (size=12301) 2024-11-24T20:49:28,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-24T20:49:28,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:28,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:28,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:28,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:28,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:28,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-24T20:49:28,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:29,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:29,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481429015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:29,018 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,018 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,018 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:29,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:29,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481429015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:29,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:29,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481429015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:29,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:29,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:29,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481429015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:29,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:29,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:29,019 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481429015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:29,019 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,019 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:29,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:29,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-24T20:49:29,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:29,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:29,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:29,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:29,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:29,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/b4d71f946e34450bb317c3b2754f0098 2024-11-24T20:49:29,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/5be49361c15541ec9b8ca309d37d2a30 is 50, key is test_row_0/C:col10/1732481368367/Put/seqid=0 2024-11-24T20:49:29,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-24T20:49:29,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:29,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:29,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:29,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:29,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:29,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742445_1621 (size=12301) 2024-11-24T20:49:29,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/5be49361c15541ec9b8ca309d37d2a30 2024-11-24T20:49:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-24T20:49:29,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/13d06b00d9584dc6be8b5c6780b31cb9 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/13d06b00d9584dc6be8b5c6780b31cb9 2024-11-24T20:49:29,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/13d06b00d9584dc6be8b5c6780b31cb9, entries=200, sequenceid=333, filesize=39.0 K 2024-11-24T20:49:29,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/b4d71f946e34450bb317c3b2754f0098 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/b4d71f946e34450bb317c3b2754f0098 2024-11-24T20:49:29,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/b4d71f946e34450bb317c3b2754f0098, entries=150, sequenceid=333, filesize=12.0 K 2024-11-24T20:49:29,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/5be49361c15541ec9b8ca309d37d2a30 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/5be49361c15541ec9b8ca309d37d2a30 2024-11-24T20:49:29,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/5be49361c15541ec9b8ca309d37d2a30, entries=150, sequenceid=333, filesize=12.0 K 2024-11-24T20:49:29,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 5eaa0424086a20a4fd6f784698d02aeb in 865ms, sequenceid=333, compaction requested=true 2024-11-24T20:49:29,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:29,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaa0424086a20a4fd6f784698d02aeb:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:49:29,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:49:29,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaa0424086a20a4fd6f784698d02aeb:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:49:29,233 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-24T20:49:29,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:49:29,233 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-24T20:49:29,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaa0424086a20a4fd6f784698d02aeb:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:49:29,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:49:29,234 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 165399 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-24T20:49:29,234 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 5eaa0424086a20a4fd6f784698d02aeb/A is initiating minor compaction (all files) 2024-11-24T20:49:29,234 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaa0424086a20a4fd6f784698d02aeb/A in TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:29,234 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/97f985660c054afd9761eda24434460d, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/6e68ce05ee3b43cb9fca6c59a5028835, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/6b03291bf0114c6a91dcdaf42b8db7ae, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/6502a2eb5c9948249d908af1335416a0, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/13d06b00d9584dc6be8b5c6780b31cb9] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp, totalSize=161.5 K 2024-11-24T20:49:29,234 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:29,234 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. files: [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/97f985660c054afd9761eda24434460d, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/6e68ce05ee3b43cb9fca6c59a5028835, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/6b03291bf0114c6a91dcdaf42b8db7ae, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/6502a2eb5c9948249d908af1335416a0, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/13d06b00d9584dc6be8b5c6780b31cb9] 2024-11-24T20:49:29,235 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61935 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-24T20:49:29,235 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 5eaa0424086a20a4fd6f784698d02aeb/B is initiating minor compaction (all files) 2024-11-24T20:49:29,235 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaa0424086a20a4fd6f784698d02aeb/B in TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:29,235 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/9fe001aa38064fbf92b6776ca504ad5c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/307b323a078546afa2753b764e1b784b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/e11edd69e60c4509ac97231c71dd04d2, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/b365c601758148b79f4da435b6af1fea, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/b4d71f946e34450bb317c3b2754f0098] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp, totalSize=60.5 K 2024-11-24T20:49:29,235 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97f985660c054afd9761eda24434460d, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732481362678 2024-11-24T20:49:29,235 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fe001aa38064fbf92b6776ca504ad5c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732481362678 2024-11-24T20:49:29,235 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e68ce05ee3b43cb9fca6c59a5028835, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732481363028 2024-11-24T20:49:29,235 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 307b323a078546afa2753b764e1b784b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732481363028 2024-11-24T20:49:29,236 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b03291bf0114c6a91dcdaf42b8db7ae, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732481363373 2024-11-24T20:49:29,236 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting e11edd69e60c4509ac97231c71dd04d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732481363373 2024-11-24T20:49:29,236 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting b365c601758148b79f4da435b6af1fea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732481365555 2024-11-24T20:49:29,236 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6502a2eb5c9948249d908af1335416a0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732481365555 2024-11-24T20:49:29,236 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting b4d71f946e34450bb317c3b2754f0098, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732481367731 2024-11-24T20:49:29,236 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13d06b00d9584dc6be8b5c6780b31cb9, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732481367731 2024-11-24T20:49:29,247 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaa0424086a20a4fd6f784698d02aeb#B#compaction#526 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:49:29,247 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/8f8e4e8afa73436ba1f4b3ace91210dd is 50, key is test_row_0/B:col10/1732481368367/Put/seqid=0 2024-11-24T20:49:29,249 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:29,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742446_1622 (size=13051) 2024-11-24T20:49:29,252 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411247420f1ac801042dab570ea8903a653d6_5eaa0424086a20a4fd6f784698d02aeb store=[table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:29,255 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411247420f1ac801042dab570ea8903a653d6_5eaa0424086a20a4fd6f784698d02aeb, store=[table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:29,255 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411247420f1ac801042dab570ea8903a653d6_5eaa0424086a20a4fd6f784698d02aeb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:29,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742447_1623 (size=4469) 2024-11-24T20:49:29,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-24T20:49:29,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:29,366 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing 5eaa0424086a20a4fd6f784698d02aeb 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-24T20:49:29,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124aae2dfa38bd14e268a819a22296a2158_5eaa0424086a20a4fd6f784698d02aeb is 50, key is test_row_0/A:col10/1732481368369/Put/seqid=0 2024-11-24T20:49:29,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742448_1624 (size=12454) 2024-11-24T20:49:29,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:49:29,401 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124aae2dfa38bd14e268a819a22296a2158_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124aae2dfa38bd14e268a819a22296a2158_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:29,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/27bd51bcbf3f40cf8b08881b55c28bcd, store: [table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:29,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/27bd51bcbf3f40cf8b08881b55c28bcd is 175, key is test_row_0/A:col10/1732481368369/Put/seqid=0 2024-11-24T20:49:29,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742449_1625 (size=31255) 2024-11-24T20:49:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:29,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:29,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:29,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481429543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:29,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:29,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481429544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:29,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:29,549 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:29,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:29,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481429545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:29,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:29,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481429545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:29,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:29,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:29,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:29,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481429546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:29,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:29,654 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/8f8e4e8afa73436ba1f4b3ace91210dd as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/8f8e4e8afa73436ba1f4b3ace91210dd 2024-11-24T20:49:29,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:29,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481429655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:29,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:29,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481429655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:29,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:29,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:29,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:29,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481429665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:29,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:29,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481429665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:29,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:29,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:29,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:29,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481429666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:29,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:29,670 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaa0424086a20a4fd6f784698d02aeb#A#compaction#527 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:49:29,671 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/1b78c57d073a4bdf838f7cbe42e883a7 is 175, key is test_row_0/A:col10/1732481368367/Put/seqid=0 2024-11-24T20:49:29,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742450_1626 (size=32005) 2024-11-24T20:49:29,694 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5eaa0424086a20a4fd6f784698d02aeb/B of 5eaa0424086a20a4fd6f784698d02aeb into 8f8e4e8afa73436ba1f4b3ace91210dd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:49:29,694 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:29,694 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., storeName=5eaa0424086a20a4fd6f784698d02aeb/B, priority=11, startTime=1732481369233; duration=0sec 2024-11-24T20:49:29,694 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:49:29,694 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaa0424086a20a4fd6f784698d02aeb:B 2024-11-24T20:49:29,694 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-24T20:49:29,695 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 86135 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-24T20:49:29,696 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 5eaa0424086a20a4fd6f784698d02aeb/C is initiating minor compaction (all files) 2024-11-24T20:49:29,696 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaa0424086a20a4fd6f784698d02aeb/C in TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:29,696 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/a61f292c19b9493bb737526258e9b3d9, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/c8864fc6f805484ebda6507309de054d, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/e0da9490eb15404b983e13a19e62986c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/66a0efed06434c1aac3ee9729979b17f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/2a5e580344374f19bea2a2240f26d29e, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/04b034d0e2f24b4996c1ccd81d47a0c9, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/5be49361c15541ec9b8ca309d37d2a30] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp, totalSize=84.1 K 2024-11-24T20:49:29,696 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting a61f292c19b9493bb737526258e9b3d9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732481360364 2024-11-24T20:49:29,696 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting c8864fc6f805484ebda6507309de054d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732481361528 2024-11-24T20:49:29,696 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting e0da9490eb15404b983e13a19e62986c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732481362678 2024-11-24T20:49:29,697 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 66a0efed06434c1aac3ee9729979b17f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732481363028 2024-11-24T20:49:29,697 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a5e580344374f19bea2a2240f26d29e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732481363373 2024-11-24T20:49:29,697 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 04b034d0e2f24b4996c1ccd81d47a0c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732481365555 2024-11-24T20:49:29,697 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 5be49361c15541ec9b8ca309d37d2a30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732481367731 2024-11-24T20:49:29,705 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaa0424086a20a4fd6f784698d02aeb#C#compaction#529 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:49:29,705 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/b36b49056bff4eba918ca86da20fa6ee is 50, key is test_row_0/C:col10/1732481368367/Put/seqid=0 2024-11-24T20:49:29,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742451_1627 (size=13017) 2024-11-24T20:49:29,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-24T20:49:29,819 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=350, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/27bd51bcbf3f40cf8b08881b55c28bcd 2024-11-24T20:49:29,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/00a4bacf34594a33a803ddb4180102eb is 50, key is test_row_0/B:col10/1732481368369/Put/seqid=0 2024-11-24T20:49:29,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742452_1628 (size=12301) 2024-11-24T20:49:29,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:29,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481429865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:29,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:29,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481429865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:29,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:29,868 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,868 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,868 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:29,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:29,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481429874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:29,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:29,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:29,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481429875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:29,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:29,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:29,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481429875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:29,881 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,881 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:29,881 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:30,077 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/1b78c57d073a4bdf838f7cbe42e883a7 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/1b78c57d073a4bdf838f7cbe42e883a7 2024-11-24T20:49:30,080 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5eaa0424086a20a4fd6f784698d02aeb/A of 5eaa0424086a20a4fd6f784698d02aeb into 1b78c57d073a4bdf838f7cbe42e883a7(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:49:30,080 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:30,080 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., storeName=5eaa0424086a20a4fd6f784698d02aeb/A, priority=11, startTime=1732481369233; duration=0sec 2024-11-24T20:49:30,080 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:49:30,080 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaa0424086a20a4fd6f784698d02aeb:A 2024-11-24T20:49:30,112 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/b36b49056bff4eba918ca86da20fa6ee as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/b36b49056bff4eba918ca86da20fa6ee 2024-11-24T20:49:30,117 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 5eaa0424086a20a4fd6f784698d02aeb/C of 5eaa0424086a20a4fd6f784698d02aeb into b36b49056bff4eba918ca86da20fa6ee(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:49:30,117 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:30,117 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., storeName=5eaa0424086a20a4fd6f784698d02aeb/C, priority=9, startTime=1732481369233; duration=0sec 2024-11-24T20:49:30,117 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:49:30,117 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaa0424086a20a4fd6f784698d02aeb:C 2024-11-24T20:49:30,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:30,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481430175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:30,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:30,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481430175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:30,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:30,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:30,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:30,176 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:30,176 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:30,177 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:30,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:30,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481430185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:30,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:30,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:30,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:30,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:30,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481430185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:30,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:30,188 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:30,188 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:30,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481430185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:30,188 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:30,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:30,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:30,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:30,257 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/00a4bacf34594a33a803ddb4180102eb 2024-11-24T20:49:30,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/e4a8779492e4437ca6d022239be35768 is 50, key is test_row_0/C:col10/1732481368369/Put/seqid=0 2024-11-24T20:49:30,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742453_1629 (size=12301) 2024-11-24T20:49:30,667 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/e4a8779492e4437ca6d022239be35768 2024-11-24T20:49:30,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/27bd51bcbf3f40cf8b08881b55c28bcd as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/27bd51bcbf3f40cf8b08881b55c28bcd 2024-11-24T20:49:30,682 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/27bd51bcbf3f40cf8b08881b55c28bcd, entries=150, sequenceid=350, filesize=30.5 K 2024-11-24T20:49:30,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/00a4bacf34594a33a803ddb4180102eb as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/00a4bacf34594a33a803ddb4180102eb 2024-11-24T20:49:30,687 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/00a4bacf34594a33a803ddb4180102eb, entries=150, sequenceid=350, filesize=12.0 K 2024-11-24T20:49:30,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/e4a8779492e4437ca6d022239be35768 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/e4a8779492e4437ca6d022239be35768 2024-11-24T20:49:30,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:30,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481430685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:30,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:30,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:30,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:30,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:30,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481430686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:30,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:30,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:30,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:30,692 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/e4a8779492e4437ca6d022239be35768, entries=150, sequenceid=350, filesize=12.0 K 2024-11-24T20:49:30,693 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 5eaa0424086a20a4fd6f784698d02aeb in 1327ms, sequenceid=350, compaction requested=false 2024-11-24T20:49:30,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:30,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:30,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-24T20:49:30,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-24T20:49:30,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:30,698 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T20:49:30,698 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5eaa0424086a20a4fd6f784698d02aeb 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-24T20:49:30,699 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-24T20:49:30,699 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1060 sec 2024-11-24T20:49:30,701 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 2.1140 sec 2024-11-24T20:49:30,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124234b80ba2c87447dac28a80a1e8b054c_5eaa0424086a20a4fd6f784698d02aeb is 50, key is test_row_0/A:col10/1732481369543/Put/seqid=0 2024-11-24T20:49:30,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:30,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481430720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:30,724 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:30,725 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:30,725 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:30,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:30,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481430722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:30,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:30,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:30,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:30,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:30,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481430724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:30,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:30,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:30,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:30,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-24T20:49:30,735 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:49:30,736 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T20:49:30,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees 2024-11-24T20:49:30,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-24T20:49:30,737 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T20:49:30,738 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T20:49:30,738 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T20:49:30,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742454_1630 (size=14994) 2024-11-24T20:49:30,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:30,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481430835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:30,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:30,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481430835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:30,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:30,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481430835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:30,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-24T20:49:30,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-24T20:49:30,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:30,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:30,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:30,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:30,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:30,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:31,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-24T20:49:31,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:31,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:31,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:31,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:31,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:31,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481431045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:31,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:31,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:31,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:31,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481431045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:31,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:31,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:31,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:31,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:31,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481431046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:31,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:31,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:31,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-24T20:49:31,146 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:49:31,148 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124234b80ba2c87447dac28a80a1e8b054c_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124234b80ba2c87447dac28a80a1e8b054c_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:31,149 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/1efcc41f036d450da22348dc681ce668, store: [table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:31,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/1efcc41f036d450da22348dc681ce668 is 175, key is test_row_0/A:col10/1732481369543/Put/seqid=0 2024-11-24T20:49:31,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742455_1631 (size=39949) 2024-11-24T20:49:31,174 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=373, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/1efcc41f036d450da22348dc681ce668 2024-11-24T20:49:31,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/efe56b6a248c41bda71307f77175d6c3 is 50, key is test_row_0/B:col10/1732481369543/Put/seqid=0 2024-11-24T20:49:31,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-24T20:49:31,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:31,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:31,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:31,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:31,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:31,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:31,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742456_1632 (size=12301) 2024-11-24T20:49:31,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-24T20:49:31,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:31,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:31,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:31,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:31,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:31,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:31,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481431356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:31,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:31,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:31,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481431356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:31,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:31,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:31,359 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:31,359 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:31,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:31,359 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:31,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481431356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:31,359 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:31,359 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:31,359 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:31,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-24T20:49:31,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-24T20:49:31,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:31,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:31,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:31,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:31,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:31,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/efe56b6a248c41bda71307f77175d6c3 2024-11-24T20:49:31,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/5929824314954ba5a2c2188e2df333f8 is 50, key is test_row_0/C:col10/1732481369543/Put/seqid=0 2024-11-24T20:49:31,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742457_1633 (size=12301) 2024-11-24T20:49:31,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-24T20:49:31,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:31,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:31,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:31,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:31,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:31,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:31,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:31,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:31,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40618 deadline: 1732481431705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:31,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40572 deadline: 1732481431705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:31,708 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_1' in 5eaa0424086a20a4fd6f784698d02aeb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:49:31,708 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:31,708 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:31,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:31,709 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 20b981ef5c27,34625,1732481182291 for 'test_row_2' in 5eaa0424086a20a4fd6f784698d02aeb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T20:49:31,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:31,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:31,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:31,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-24T20:49:31,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:31,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:31,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:31,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:31,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:31,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:31,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:31,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:31,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40600 deadline: 1732481431865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:31,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40558 deadline: 1732481431865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:31,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T20:49:31,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:31,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:31,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34625 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40580 deadline: 1732481431865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:31,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:31,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:31,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:31,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:31,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 , the old value is region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:31,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T20:49:31,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., hostname=20b981ef5c27,34625,1732481182291, seqNum=5 because the exception is null or not the one we care about 2024-11-24T20:49:31,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-24T20:49:31,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-24T20:49:31,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:31,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:31,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:31,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:31,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:31,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T20:49:32,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/5929824314954ba5a2c2188e2df333f8 2024-11-24T20:49:32,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/1efcc41f036d450da22348dc681ce668 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/1efcc41f036d450da22348dc681ce668 2024-11-24T20:49:32,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/1efcc41f036d450da22348dc681ce668, entries=200, sequenceid=373, filesize=39.0 K 2024-11-24T20:49:32,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/efe56b6a248c41bda71307f77175d6c3 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/efe56b6a248c41bda71307f77175d6c3 2024-11-24T20:49:32,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/efe56b6a248c41bda71307f77175d6c3, entries=150, sequenceid=373, filesize=12.0 K 2024-11-24T20:49:32,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/5929824314954ba5a2c2188e2df333f8 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/5929824314954ba5a2c2188e2df333f8 2024-11-24T20:49:32,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/5929824314954ba5a2c2188e2df333f8, entries=150, sequenceid=373, filesize=12.0 K 2024-11-24T20:49:32,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 5eaa0424086a20a4fd6f784698d02aeb in 1349ms, sequenceid=373, compaction requested=true 2024-11-24T20:49:32,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:32,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaa0424086a20a4fd6f784698d02aeb:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T20:49:32,048 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:49:32,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:49:32,048 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:49:32,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaa0424086a20a4fd6f784698d02aeb:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T20:49:32,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:49:32,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5eaa0424086a20a4fd6f784698d02aeb:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T20:49:32,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:49:32,048 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:49:32,048 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:49:32,048 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 5eaa0424086a20a4fd6f784698d02aeb/B is initiating minor compaction (all files) 2024-11-24T20:49:32,048 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1541): 5eaa0424086a20a4fd6f784698d02aeb/A is initiating minor compaction (all files) 2024-11-24T20:49:32,048 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaa0424086a20a4fd6f784698d02aeb/A in TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:32,048 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaa0424086a20a4fd6f784698d02aeb/B in TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:32,049 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/1b78c57d073a4bdf838f7cbe42e883a7, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/27bd51bcbf3f40cf8b08881b55c28bcd, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/1efcc41f036d450da22348dc681ce668] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp, totalSize=100.8 K 2024-11-24T20:49:32,049 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/8f8e4e8afa73436ba1f4b3ace91210dd, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/00a4bacf34594a33a803ddb4180102eb, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/efe56b6a248c41bda71307f77175d6c3] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp, totalSize=36.8 K 2024-11-24T20:49:32,049 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:32,049 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. files: [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/1b78c57d073a4bdf838f7cbe42e883a7, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/27bd51bcbf3f40cf8b08881b55c28bcd, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/1efcc41f036d450da22348dc681ce668] 2024-11-24T20:49:32,049 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f8e4e8afa73436ba1f4b3ace91210dd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732481367731 2024-11-24T20:49:32,049 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 00a4bacf34594a33a803ddb4180102eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732481368369 2024-11-24T20:49:32,049 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b78c57d073a4bdf838f7cbe42e883a7, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732481367731 2024-11-24T20:49:32,050 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27bd51bcbf3f40cf8b08881b55c28bcd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732481368369 2024-11-24T20:49:32,050 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting efe56b6a248c41bda71307f77175d6c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732481369542 2024-11-24T20:49:32,050 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1efcc41f036d450da22348dc681ce668, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732481369542 2024-11-24T20:49:32,057 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:32,057 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaa0424086a20a4fd6f784698d02aeb#B#compaction#535 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:49:32,058 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/3bf5374916fa4cbb9129f7198b298d3c is 50, key is test_row_0/B:col10/1732481369543/Put/seqid=0 2024-11-24T20:49:32,061 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124312c4a11aa024444b5000de2c85de9bc_5eaa0424086a20a4fd6f784698d02aeb store=[table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:32,063 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124312c4a11aa024444b5000de2c85de9bc_5eaa0424086a20a4fd6f784698d02aeb, store=[table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:32,063 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124312c4a11aa024444b5000de2c85de9bc_5eaa0424086a20a4fd6f784698d02aeb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:32,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742459_1635 (size=4469) 2024-11-24T20:49:32,088 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaa0424086a20a4fd6f784698d02aeb#A#compaction#536 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:49:32,088 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/3de5a4c21d0045ea8c92372a3a8282ba is 175, key is test_row_0/A:col10/1732481369543/Put/seqid=0 2024-11-24T20:49:32,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742458_1634 (size=13153) 2024-11-24T20:49:32,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742460_1636 (size=32107) 2024-11-24T20:49:32,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34625 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-24T20:49:32,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:32,118 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2902): Flushing 5eaa0424086a20a4fd6f784698d02aeb 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-24T20:49:32,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411248339d37634fe4e81b1a491ed75699711_5eaa0424086a20a4fd6f784698d02aeb is 50, key is test_row_0/A:col10/1732481370723/Put/seqid=0 2024-11-24T20:49:32,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742461_1637 (size=12454) 2024-11-24T20:49:32,453 INFO [Thread-2413 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2413. 2024-11-24T20:49:32,453 DEBUG [Thread-2413 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:49:32,453 DEBUG [Thread-2413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:32,453 DEBUG [Thread-2413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:32,453 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:49:32,455 INFO [Thread-2409 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2409. 2024-11-24T20:49:32,455 DEBUG [Thread-2409 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:49:32,455 DEBUG [Thread-2409 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:32,455 INFO [Thread-2415 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2415. 2024-11-24T20:49:32,455 DEBUG [Thread-2415 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:49:32,455 DEBUG [Thread-2409 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:32,455 DEBUG [Thread-2415 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:32,455 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:49:32,455 DEBUG [Thread-2415 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:32,455 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:49:32,456 INFO [Thread-2407 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2407. 2024-11-24T20:49:32,456 DEBUG [Thread-2407 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:49:32,456 DEBUG [Thread-2407 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:32,456 DEBUG [Thread-2407 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:32,456 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:49:32,458 INFO [Thread-2411 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2411. 2024-11-24T20:49:32,458 DEBUG [Thread-2411 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:49:32,458 DEBUG [Thread-2411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:32,458 DEBUG [Thread-2411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:32,458 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:49:32,493 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/3bf5374916fa4cbb9129f7198b298d3c as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/3bf5374916fa4cbb9129f7198b298d3c 2024-11-24T20:49:32,497 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/3de5a4c21d0045ea8c92372a3a8282ba as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/3de5a4c21d0045ea8c92372a3a8282ba 2024-11-24T20:49:32,497 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eaa0424086a20a4fd6f784698d02aeb/B of 5eaa0424086a20a4fd6f784698d02aeb into 3bf5374916fa4cbb9129f7198b298d3c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:49:32,497 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:32,497 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., storeName=5eaa0424086a20a4fd6f784698d02aeb/B, priority=13, startTime=1732481372048; duration=0sec 2024-11-24T20:49:32,497 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T20:49:32,497 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaa0424086a20a4fd6f784698d02aeb:B 2024-11-24T20:49:32,497 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T20:49:32,497 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T20:49:32,498 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1541): 5eaa0424086a20a4fd6f784698d02aeb/C is initiating minor compaction (all files) 2024-11-24T20:49:32,498 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5eaa0424086a20a4fd6f784698d02aeb/C in TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:32,498 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/b36b49056bff4eba918ca86da20fa6ee, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/e4a8779492e4437ca6d022239be35768, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/5929824314954ba5a2c2188e2df333f8] into tmpdir=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp, totalSize=36.7 K 2024-11-24T20:49:32,498 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting b36b49056bff4eba918ca86da20fa6ee, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732481367731 2024-11-24T20:49:32,498 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting e4a8779492e4437ca6d022239be35768, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732481368369 2024-11-24T20:49:32,498 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] compactions.Compactor(225): Compacting 5929824314954ba5a2c2188e2df333f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732481369542 2024-11-24T20:49:32,500 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eaa0424086a20a4fd6f784698d02aeb/A of 5eaa0424086a20a4fd6f784698d02aeb into 3de5a4c21d0045ea8c92372a3a8282ba(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:49:32,500 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:32,501 INFO [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., storeName=5eaa0424086a20a4fd6f784698d02aeb/A, priority=13, startTime=1732481372047; duration=0sec 2024-11-24T20:49:32,501 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:49:32,501 DEBUG [RS:0;20b981ef5c27:34625-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaa0424086a20a4fd6f784698d02aeb:A 2024-11-24T20:49:32,503 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5eaa0424086a20a4fd6f784698d02aeb#C#compaction#538 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T20:49:32,503 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/52329df339a2413f83041cca9da3ffc6 is 50, key is test_row_0/C:col10/1732481369543/Put/seqid=0 2024-11-24T20:49:32,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742462_1638 (size=13119) 2024-11-24T20:49:32,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:49:32,533 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411248339d37634fe4e81b1a491ed75699711_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248339d37634fe4e81b1a491ed75699711_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:32,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/ae570a9ba4ea475eb375eb181a1dfdb5, store: [table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:32,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/ae570a9ba4ea475eb375eb181a1dfdb5 is 175, key is test_row_0/A:col10/1732481370723/Put/seqid=0 2024-11-24T20:49:32,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742463_1639 (size=31255) 2024-11-24T20:49:32,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34625 {}] regionserver.HRegion(8855): Flush requested on 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:32,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. as already flushing 2024-11-24T20:49:32,876 INFO [Thread-2402 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2402. 2024-11-24T20:49:32,876 INFO [Thread-2398 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2398. 2024-11-24T20:49:32,876 DEBUG [Thread-2402 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:49:32,876 DEBUG [Thread-2402 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:32,876 DEBUG [Thread-2398 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:49:32,876 DEBUG [Thread-2398 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:32,877 DEBUG [Thread-2402 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:32,877 DEBUG [Thread-2398 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:32,877 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:49:32,877 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:49:32,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-24T20:49:32,887 INFO [Thread-2396 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2396. 2024-11-24T20:49:32,887 DEBUG [Thread-2396 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:49:32,887 DEBUG [Thread-2396 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:32,887 DEBUG [Thread-2396 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:32,887 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:49:32,915 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/52329df339a2413f83041cca9da3ffc6 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/52329df339a2413f83041cca9da3ffc6 2024-11-24T20:49:32,919 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5eaa0424086a20a4fd6f784698d02aeb/C of 5eaa0424086a20a4fd6f784698d02aeb into 52329df339a2413f83041cca9da3ffc6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T20:49:32,919 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:32,919 INFO [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., storeName=5eaa0424086a20a4fd6f784698d02aeb/C, priority=13, startTime=1732481372048; duration=0sec 2024-11-24T20:49:32,919 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T20:49:32,919 DEBUG [RS:0;20b981ef5c27:34625-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5eaa0424086a20a4fd6f784698d02aeb:C 2024-11-24T20:49:32,938 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=389, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/ae570a9ba4ea475eb375eb181a1dfdb5 2024-11-24T20:49:32,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/cf0d030691d8417ea1ae472a600a6f14 is 50, key is test_row_0/B:col10/1732481370723/Put/seqid=0 2024-11-24T20:49:32,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742464_1640 (size=12301) 2024-11-24T20:49:33,349 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/cf0d030691d8417ea1ae472a600a6f14 2024-11-24T20:49:33,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/ad319cf16efd40f081d82a752027548e is 50, key is test_row_0/C:col10/1732481370723/Put/seqid=0 2024-11-24T20:49:33,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742465_1641 (size=12301) 2024-11-24T20:49:33,727 INFO [Thread-2400 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2400. 2024-11-24T20:49:33,727 DEBUG [Thread-2400 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:49:33,727 DEBUG [Thread-2400 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:33,727 DEBUG [Thread-2400 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:33,727 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:49:33,738 INFO [Thread-2404 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2404. 2024-11-24T20:49:33,738 DEBUG [Thread-2404 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T20:49:33,738 DEBUG [Thread-2404 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:33,738 DEBUG [Thread-2404 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:33,738 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:49:33,760 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/ad319cf16efd40f081d82a752027548e 2024-11-24T20:49:33,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/ae570a9ba4ea475eb375eb181a1dfdb5 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/ae570a9ba4ea475eb375eb181a1dfdb5 2024-11-24T20:49:33,776 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/ae570a9ba4ea475eb375eb181a1dfdb5, entries=150, sequenceid=389, filesize=30.5 K 2024-11-24T20:49:33,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/cf0d030691d8417ea1ae472a600a6f14 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/cf0d030691d8417ea1ae472a600a6f14 2024-11-24T20:49:33,780 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/cf0d030691d8417ea1ae472a600a6f14, entries=150, sequenceid=389, filesize=12.0 K 2024-11-24T20:49:33,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/ad319cf16efd40f081d82a752027548e as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/ad319cf16efd40f081d82a752027548e 2024-11-24T20:49:33,785 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/ad319cf16efd40f081d82a752027548e, entries=150, sequenceid=389, filesize=12.0 K 2024-11-24T20:49:33,786 INFO [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=33.54 KB/34350 for 5eaa0424086a20a4fd6f784698d02aeb in 1668ms, sequenceid=389, compaction requested=false 2024-11-24T20:49:33,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2603): Flush status journal for 5eaa0424086a20a4fd6f784698d02aeb: 2024-11-24T20:49:33,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:33,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/20b981ef5c27:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=173 2024-11-24T20:49:33,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster(4169): Remote procedure done, pid=173 2024-11-24T20:49:33,788 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-24T20:49:33,788 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0490 sec 2024-11-24T20:49:33,790 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees in 3.0530 sec 2024-11-24T20:49:34,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-24T20:49:34,905 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:49:34,905 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T20:49:34,905 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-24T20:49:34,905 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-24T20:49:34,905 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-24T20:49:34,905 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-24T20:49:34,905 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-24T20:49:34,905 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T20:49:34,905 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3705 2024-11-24T20:49:34,905 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3482 2024-11-24T20:49:34,905 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3509 2024-11-24T20:49:34,905 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3727 2024-11-24T20:49:34,905 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3489 2024-11-24T20:49:34,905 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T20:49:34,905 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T20:49:34,906 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T20:49:34,906 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:34,906 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:34,906 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:49:34,906 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T20:49:34,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T20:49:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-24T20:49:34,910 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732481374910"}]},"ts":"1732481374910"} 2024-11-24T20:49:34,911 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T20:49:34,911 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T20:49:34,913 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T20:49:34,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=176, ppid=175, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eaa0424086a20a4fd6f784698d02aeb, UNASSIGN}] 2024-11-24T20:49:34,916 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=176, ppid=175, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eaa0424086a20a4fd6f784698d02aeb, UNASSIGN 2024-11-24T20:49:34,916 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=176 updating hbase:meta row=5eaa0424086a20a4fd6f784698d02aeb, regionState=CLOSING, regionLocation=20b981ef5c27,34625,1732481182291 2024-11-24T20:49:34,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=176, ppid=175, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eaa0424086a20a4fd6f784698d02aeb, UNASSIGN because future has completed 2024-11-24T20:49:34,918 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T20:49:34,918 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=177, ppid=176, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291}] 2024-11-24T20:49:35,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-24T20:49:35,071 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] handler.UnassignRegionHandler(122): Close 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:35,071 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T20:49:35,071 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1722): Closing 5eaa0424086a20a4fd6f784698d02aeb, disabling compactions & flushes 2024-11-24T20:49:35,071 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:35,071 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:35,071 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. after waiting 0 ms 2024-11-24T20:49:35,071 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:35,071 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(2902): Flushing 5eaa0424086a20a4fd6f784698d02aeb 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-24T20:49:35,080 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411246ceb19ded5b74301bb5471f590a8e472_5eaa0424086a20a4fd6f784698d02aeb is 50, key is test_row_1/A:col10/1732481373725/Put/seqid=0 2024-11-24T20:49:35,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742466_1642 (size=9914) 2024-11-24T20:49:35,089 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T20:49:35,092 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411246ceb19ded5b74301bb5471f590a8e472_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246ceb19ded5b74301bb5471f590a8e472_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:35,093 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/8b8634813ab74581bce180b044857fae, store: [table=TestAcidGuarantees family=A region=5eaa0424086a20a4fd6f784698d02aeb] 2024-11-24T20:49:35,093 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/8b8634813ab74581bce180b044857fae is 175, key is test_row_1/A:col10/1732481373725/Put/seqid=0 2024-11-24T20:49:35,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742467_1643 (size=22561) 2024-11-24T20:49:35,097 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=400, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/8b8634813ab74581bce180b044857fae 2024-11-24T20:49:35,104 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/ab4f249416694416b206a935fd647c0d is 50, key is test_row_1/B:col10/1732481373725/Put/seqid=0 2024-11-24T20:49:35,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742468_1644 (size=9857) 2024-11-24T20:49:35,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-24T20:49:35,513 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/ab4f249416694416b206a935fd647c0d 2024-11-24T20:49:35,527 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/3b781899142d440eb7c80469f5a54802 is 50, key is test_row_1/C:col10/1732481373725/Put/seqid=0 2024-11-24T20:49:35,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742469_1645 (size=9857) 2024-11-24T20:49:35,531 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/3b781899142d440eb7c80469f5a54802 2024-11-24T20:49:35,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-24T20:49:35,536 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/A/8b8634813ab74581bce180b044857fae as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/8b8634813ab74581bce180b044857fae 2024-11-24T20:49:35,540 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/8b8634813ab74581bce180b044857fae, entries=100, sequenceid=400, filesize=22.0 K 2024-11-24T20:49:35,540 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/B/ab4f249416694416b206a935fd647c0d as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/ab4f249416694416b206a935fd647c0d 2024-11-24T20:49:35,544 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/ab4f249416694416b206a935fd647c0d, entries=100, sequenceid=400, filesize=9.6 K 2024-11-24T20:49:35,546 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/.tmp/C/3b781899142d440eb7c80469f5a54802 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/3b781899142d440eb7c80469f5a54802 2024-11-24T20:49:35,550 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/3b781899142d440eb7c80469f5a54802, entries=100, sequenceid=400, filesize=9.6 K 2024-11-24T20:49:35,550 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 5eaa0424086a20a4fd6f784698d02aeb in 479ms, sequenceid=400, compaction requested=true 2024-11-24T20:49:35,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/50c705f2924d4f07b3dc787dc9f495db, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/d53712b6979a473d8baa5a4a4a3dcbd2, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/97306ac4d7af49d2b9ac835692cb2a69, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/4005cd0f55394ba0bc0680198dbe9323, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/81aab15de1be48d4b1998036cb19a7ae, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/3816d39cc27d4cfd8c3ca6861a69c203, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/c5a8a9213a7c4fa4a73ffc0cb77a4b78, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/6c7d009d7cdc4369b718e64ef28a0982, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/b93342339047408e82f7fb869aa435f4, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/22968dc3c8734110b69e52867e015885, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/5edf133c85a94b13ba21cb51189276f7, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/7307b0dca80044fc8f30fac410bab605, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/45c7009754f04f58a4424acf1781d6d2, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/662c8328953a4772901dcf404d93e895, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/4f8ae702c1a24cd6985a38e5d6b243b3, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/853a03f53b834f9db46b5a1c4b00068a, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/f0f0fe87494544e19befac2ae61602f6, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/97f985660c054afd9761eda24434460d, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/6e68ce05ee3b43cb9fca6c59a5028835, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/6b03291bf0114c6a91dcdaf42b8db7ae, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/6502a2eb5c9948249d908af1335416a0, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/13d06b00d9584dc6be8b5c6780b31cb9, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/1b78c57d073a4bdf838f7cbe42e883a7, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/27bd51bcbf3f40cf8b08881b55c28bcd, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/1efcc41f036d450da22348dc681ce668] to archive 2024-11-24T20:49:35,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T20:49:35,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/50c705f2924d4f07b3dc787dc9f495db to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/50c705f2924d4f07b3dc787dc9f495db 2024-11-24T20:49:35,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/d53712b6979a473d8baa5a4a4a3dcbd2 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/d53712b6979a473d8baa5a4a4a3dcbd2 2024-11-24T20:49:35,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/97306ac4d7af49d2b9ac835692cb2a69 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/97306ac4d7af49d2b9ac835692cb2a69 2024-11-24T20:49:35,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/4005cd0f55394ba0bc0680198dbe9323 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/4005cd0f55394ba0bc0680198dbe9323 2024-11-24T20:49:35,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/81aab15de1be48d4b1998036cb19a7ae to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/81aab15de1be48d4b1998036cb19a7ae 2024-11-24T20:49:35,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/3816d39cc27d4cfd8c3ca6861a69c203 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/3816d39cc27d4cfd8c3ca6861a69c203 2024-11-24T20:49:35,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/c5a8a9213a7c4fa4a73ffc0cb77a4b78 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/c5a8a9213a7c4fa4a73ffc0cb77a4b78 2024-11-24T20:49:35,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/6c7d009d7cdc4369b718e64ef28a0982 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/6c7d009d7cdc4369b718e64ef28a0982 2024-11-24T20:49:35,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/b93342339047408e82f7fb869aa435f4 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/b93342339047408e82f7fb869aa435f4 2024-11-24T20:49:35,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/22968dc3c8734110b69e52867e015885 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/22968dc3c8734110b69e52867e015885 2024-11-24T20:49:35,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/5edf133c85a94b13ba21cb51189276f7 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/5edf133c85a94b13ba21cb51189276f7 2024-11-24T20:49:35,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/7307b0dca80044fc8f30fac410bab605 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/7307b0dca80044fc8f30fac410bab605 2024-11-24T20:49:35,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/45c7009754f04f58a4424acf1781d6d2 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/45c7009754f04f58a4424acf1781d6d2 2024-11-24T20:49:35,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/662c8328953a4772901dcf404d93e895 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/662c8328953a4772901dcf404d93e895 2024-11-24T20:49:35,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/4f8ae702c1a24cd6985a38e5d6b243b3 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/4f8ae702c1a24cd6985a38e5d6b243b3 2024-11-24T20:49:35,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/853a03f53b834f9db46b5a1c4b00068a to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/853a03f53b834f9db46b5a1c4b00068a 2024-11-24T20:49:35,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/f0f0fe87494544e19befac2ae61602f6 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/f0f0fe87494544e19befac2ae61602f6 2024-11-24T20:49:35,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/97f985660c054afd9761eda24434460d to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/97f985660c054afd9761eda24434460d 2024-11-24T20:49:35,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/6e68ce05ee3b43cb9fca6c59a5028835 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/6e68ce05ee3b43cb9fca6c59a5028835 2024-11-24T20:49:35,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/6b03291bf0114c6a91dcdaf42b8db7ae to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/6b03291bf0114c6a91dcdaf42b8db7ae 2024-11-24T20:49:35,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/6502a2eb5c9948249d908af1335416a0 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/6502a2eb5c9948249d908af1335416a0 2024-11-24T20:49:35,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/13d06b00d9584dc6be8b5c6780b31cb9 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/13d06b00d9584dc6be8b5c6780b31cb9 2024-11-24T20:49:35,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/1b78c57d073a4bdf838f7cbe42e883a7 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/1b78c57d073a4bdf838f7cbe42e883a7 2024-11-24T20:49:35,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/27bd51bcbf3f40cf8b08881b55c28bcd to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/27bd51bcbf3f40cf8b08881b55c28bcd 2024-11-24T20:49:35,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/1efcc41f036d450da22348dc681ce668 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/1efcc41f036d450da22348dc681ce668 2024-11-24T20:49:35,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/fdbb452481ba4f27af3327ac9c35998f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/bca323f5ab0a4f9ba9bc401100bd8934, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/0cc52863a9bd469884ac42428acebf38, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/66023724b035402f848a2437976dc477, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/ea66ae124a1f4176b3d6527ea32aaa22, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/396e661b33a541a0952ff9c316e316bd, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/4636fdc347bf4af4ba525b1bd495d42e, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/5da84091243c472bbb9940c99aa6004e, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/fd033b03afb44a5687983a496d6d8e1c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/df5a65b30dea46209d1da9589a8f05a7, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/d7ea77bd9eb04f4fae87e673e7756fdd, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/8ddb602df90f4d0d88f7d6d0ef41210c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/c16c6b249f31432eb4796a3b36387ae9, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/527940924315465e8d03ef76f272541d, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/0916d1c4e0cc450c9ce911ef4e6c9b5c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/a03a2e0299ad411bb354574c438bab49, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/9fe001aa38064fbf92b6776ca504ad5c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/a65c73ba7aa24396a0d5c9266b594b47, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/307b323a078546afa2753b764e1b784b, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/e11edd69e60c4509ac97231c71dd04d2, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/b365c601758148b79f4da435b6af1fea, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/8f8e4e8afa73436ba1f4b3ace91210dd, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/b4d71f946e34450bb317c3b2754f0098, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/00a4bacf34594a33a803ddb4180102eb, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/efe56b6a248c41bda71307f77175d6c3] to archive 2024-11-24T20:49:35,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T20:49:35,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/fdbb452481ba4f27af3327ac9c35998f to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/fdbb452481ba4f27af3327ac9c35998f 2024-11-24T20:49:35,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/bca323f5ab0a4f9ba9bc401100bd8934 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/bca323f5ab0a4f9ba9bc401100bd8934 2024-11-24T20:49:35,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/0cc52863a9bd469884ac42428acebf38 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/0cc52863a9bd469884ac42428acebf38 2024-11-24T20:49:35,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/66023724b035402f848a2437976dc477 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/66023724b035402f848a2437976dc477 2024-11-24T20:49:35,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/ea66ae124a1f4176b3d6527ea32aaa22 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/ea66ae124a1f4176b3d6527ea32aaa22 2024-11-24T20:49:35,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/396e661b33a541a0952ff9c316e316bd to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/396e661b33a541a0952ff9c316e316bd 2024-11-24T20:49:35,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/4636fdc347bf4af4ba525b1bd495d42e to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/4636fdc347bf4af4ba525b1bd495d42e 2024-11-24T20:49:35,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/5da84091243c472bbb9940c99aa6004e to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/5da84091243c472bbb9940c99aa6004e 2024-11-24T20:49:35,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/fd033b03afb44a5687983a496d6d8e1c to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/fd033b03afb44a5687983a496d6d8e1c 2024-11-24T20:49:35,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/df5a65b30dea46209d1da9589a8f05a7 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/df5a65b30dea46209d1da9589a8f05a7 2024-11-24T20:49:35,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/d7ea77bd9eb04f4fae87e673e7756fdd to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/d7ea77bd9eb04f4fae87e673e7756fdd 2024-11-24T20:49:35,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/8ddb602df90f4d0d88f7d6d0ef41210c to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/8ddb602df90f4d0d88f7d6d0ef41210c 2024-11-24T20:49:35,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/c16c6b249f31432eb4796a3b36387ae9 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/c16c6b249f31432eb4796a3b36387ae9 2024-11-24T20:49:35,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/527940924315465e8d03ef76f272541d to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/527940924315465e8d03ef76f272541d 2024-11-24T20:49:35,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/0916d1c4e0cc450c9ce911ef4e6c9b5c to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/0916d1c4e0cc450c9ce911ef4e6c9b5c 2024-11-24T20:49:35,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/a03a2e0299ad411bb354574c438bab49 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/a03a2e0299ad411bb354574c438bab49 2024-11-24T20:49:35,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/9fe001aa38064fbf92b6776ca504ad5c to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/9fe001aa38064fbf92b6776ca504ad5c 2024-11-24T20:49:35,591 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/a65c73ba7aa24396a0d5c9266b594b47 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/a65c73ba7aa24396a0d5c9266b594b47 2024-11-24T20:49:35,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/307b323a078546afa2753b764e1b784b to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/307b323a078546afa2753b764e1b784b 2024-11-24T20:49:35,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/e11edd69e60c4509ac97231c71dd04d2 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/e11edd69e60c4509ac97231c71dd04d2 2024-11-24T20:49:35,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/b365c601758148b79f4da435b6af1fea to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/b365c601758148b79f4da435b6af1fea 2024-11-24T20:49:35,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/8f8e4e8afa73436ba1f4b3ace91210dd to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/8f8e4e8afa73436ba1f4b3ace91210dd 2024-11-24T20:49:35,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/b4d71f946e34450bb317c3b2754f0098 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/b4d71f946e34450bb317c3b2754f0098 2024-11-24T20:49:35,596 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/00a4bacf34594a33a803ddb4180102eb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/00a4bacf34594a33a803ddb4180102eb 2024-11-24T20:49:35,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/efe56b6a248c41bda71307f77175d6c3 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/efe56b6a248c41bda71307f77175d6c3 2024-11-24T20:49:35,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/581f505a5a2e4688a750ba3410f2ab32, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/57048206afe040f9b02d6705ea27e284, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/3c6529f9da90406e846e81dff7398983, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/dbe1d21541b144ff9c35497cb2228f65, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/61d73e81ef1549058bb12a5bd68625ce, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/94a1382d1ae54cbda1f4678a5dc3a186, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/7a921cb32cb444a3b6def39d841eb185, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/ddb44ad9d34d4d6e9eaecf002cb98c7f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/49c39a04a0ab4b66a6609de5f3c9fc70, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/63e74b1d5cdb4e4c9a82ead7d3abf047, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/09656784944f48d0bcf6890ffdfe2b7a, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/877c51204f814249a0c5a1628b1e2624, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/311f582fd2e343d3a0135d1c7d411b22, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/a61f292c19b9493bb737526258e9b3d9, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/61fa0c29f0c8462fb845c84c4d4e1bc8, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/c8864fc6f805484ebda6507309de054d, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/e0da9490eb15404b983e13a19e62986c, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/66a0efed06434c1aac3ee9729979b17f, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/2a5e580344374f19bea2a2240f26d29e, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/04b034d0e2f24b4996c1ccd81d47a0c9, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/b36b49056bff4eba918ca86da20fa6ee, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/5be49361c15541ec9b8ca309d37d2a30, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/e4a8779492e4437ca6d022239be35768, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/5929824314954ba5a2c2188e2df333f8] to archive 2024-11-24T20:49:35,599 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T20:49:35,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/581f505a5a2e4688a750ba3410f2ab32 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/581f505a5a2e4688a750ba3410f2ab32 2024-11-24T20:49:35,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/57048206afe040f9b02d6705ea27e284 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/57048206afe040f9b02d6705ea27e284 2024-11-24T20:49:35,602 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/3c6529f9da90406e846e81dff7398983 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/3c6529f9da90406e846e81dff7398983 2024-11-24T20:49:35,602 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/dbe1d21541b144ff9c35497cb2228f65 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/dbe1d21541b144ff9c35497cb2228f65 2024-11-24T20:49:35,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/61d73e81ef1549058bb12a5bd68625ce to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/61d73e81ef1549058bb12a5bd68625ce 2024-11-24T20:49:35,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/94a1382d1ae54cbda1f4678a5dc3a186 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/94a1382d1ae54cbda1f4678a5dc3a186 2024-11-24T20:49:35,605 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/7a921cb32cb444a3b6def39d841eb185 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/7a921cb32cb444a3b6def39d841eb185 2024-11-24T20:49:35,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/ddb44ad9d34d4d6e9eaecf002cb98c7f to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/ddb44ad9d34d4d6e9eaecf002cb98c7f 2024-11-24T20:49:35,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/49c39a04a0ab4b66a6609de5f3c9fc70 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/49c39a04a0ab4b66a6609de5f3c9fc70 2024-11-24T20:49:35,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/63e74b1d5cdb4e4c9a82ead7d3abf047 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/63e74b1d5cdb4e4c9a82ead7d3abf047 2024-11-24T20:49:35,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/09656784944f48d0bcf6890ffdfe2b7a to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/09656784944f48d0bcf6890ffdfe2b7a 2024-11-24T20:49:35,609 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/877c51204f814249a0c5a1628b1e2624 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/877c51204f814249a0c5a1628b1e2624 2024-11-24T20:49:35,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/311f582fd2e343d3a0135d1c7d411b22 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/311f582fd2e343d3a0135d1c7d411b22 2024-11-24T20:49:35,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/a61f292c19b9493bb737526258e9b3d9 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/a61f292c19b9493bb737526258e9b3d9 2024-11-24T20:49:35,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/61fa0c29f0c8462fb845c84c4d4e1bc8 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/61fa0c29f0c8462fb845c84c4d4e1bc8 2024-11-24T20:49:35,612 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/c8864fc6f805484ebda6507309de054d to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/c8864fc6f805484ebda6507309de054d 2024-11-24T20:49:35,613 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/e0da9490eb15404b983e13a19e62986c to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/e0da9490eb15404b983e13a19e62986c 2024-11-24T20:49:35,613 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/66a0efed06434c1aac3ee9729979b17f to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/66a0efed06434c1aac3ee9729979b17f 2024-11-24T20:49:35,614 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/2a5e580344374f19bea2a2240f26d29e to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/2a5e580344374f19bea2a2240f26d29e 2024-11-24T20:49:35,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/04b034d0e2f24b4996c1ccd81d47a0c9 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/04b034d0e2f24b4996c1ccd81d47a0c9 2024-11-24T20:49:35,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/b36b49056bff4eba918ca86da20fa6ee to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/b36b49056bff4eba918ca86da20fa6ee 2024-11-24T20:49:35,616 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/5be49361c15541ec9b8ca309d37d2a30 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/5be49361c15541ec9b8ca309d37d2a30 2024-11-24T20:49:35,617 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/e4a8779492e4437ca6d022239be35768 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/e4a8779492e4437ca6d022239be35768 2024-11-24T20:49:35,618 DEBUG [StoreCloser-TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/5929824314954ba5a2c2188e2df333f8 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/5929824314954ba5a2c2188e2df333f8 2024-11-24T20:49:35,622 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/recovered.edits/403.seqid, newMaxSeqId=403, maxSeqId=4 2024-11-24T20:49:35,622 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. 2024-11-24T20:49:35,622 DEBUG [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1676): Region close journal for 5eaa0424086a20a4fd6f784698d02aeb: Waiting for close lock at 1732481375071Running coprocessor pre-close hooks at 1732481375071Disabling compacts and flushes for region at 1732481375071Disabling writes for close at 1732481375071Obtaining lock to block concurrent updates at 1732481375071Preparing flush snapshotting stores in 5eaa0424086a20a4fd6f784698d02aeb at 1732481375071Finished memstore snapshotting TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732481375071Flushing stores of TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb. at 1732481375072 (+1 ms)Flushing 5eaa0424086a20a4fd6f784698d02aeb/A: creating writer at 1732481375072Flushing mob file 5eaa0424086a20a4fd6f784698d02aeb/A: appending metadata at 1732481375080 (+8 ms)Flushing mob file 5eaa0424086a20a4fd6f784698d02aeb/A: closing flushed file at 1732481375080Flushing 5eaa0424086a20a4fd6f784698d02aeb/A: appending metadata at 1732481375093 (+13 ms)Flushing 5eaa0424086a20a4fd6f784698d02aeb/A: closing flushed file at 1732481375093Flushing 5eaa0424086a20a4fd6f784698d02aeb/B: creating writer at 1732481375100 (+7 ms)Flushing 5eaa0424086a20a4fd6f784698d02aeb/B: appending metadata at 1732481375103 (+3 ms)Flushing 5eaa0424086a20a4fd6f784698d02aeb/B: closing flushed file at 1732481375103Flushing 5eaa0424086a20a4fd6f784698d02aeb/C: creating writer at 1732481375524 (+421 ms)Flushing 5eaa0424086a20a4fd6f784698d02aeb/C: appending metadata at 1732481375527 (+3 ms)Flushing 5eaa0424086a20a4fd6f784698d02aeb/C: closing flushed file at 1732481375527Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5450223a: reopening flushed file at 1732481375535 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4720f58e: reopening flushed file at 1732481375540 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6513d690: reopening flushed file at 1732481375544 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 5eaa0424086a20a4fd6f784698d02aeb in 479ms, sequenceid=400, compaction requested=true at 1732481375551 (+7 ms)Writing region close event to WAL at 1732481375619 (+68 ms)Running coprocessor post-close hooks at 1732481375622 (+3 ms)Closed at 1732481375622 2024-11-24T20:49:35,624 INFO [RS_CLOSE_REGION-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] handler.UnassignRegionHandler(157): Closed 5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:35,624 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=176 updating hbase:meta row=5eaa0424086a20a4fd6f784698d02aeb, regionState=CLOSED 2024-11-24T20:49:35,626 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=177, ppid=176, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 because future has completed 2024-11-24T20:49:35,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=177, resume processing ppid=176 2024-11-24T20:49:35,628 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=177, ppid=176, state=SUCCESS, hasLock=false; CloseRegionProcedure 5eaa0424086a20a4fd6f784698d02aeb, server=20b981ef5c27,34625,1732481182291 in 708 msec 2024-11-24T20:49:35,629 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=176, resume processing ppid=175 2024-11-24T20:49:35,629 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, ppid=175, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5eaa0424086a20a4fd6f784698d02aeb, UNASSIGN in 713 msec 2024-11-24T20:49:35,631 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-24T20:49:35,631 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 718 msec 2024-11-24T20:49:35,632 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732481375632"}]},"ts":"1732481375632"} 2024-11-24T20:49:35,633 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T20:49:35,634 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T20:49:35,635 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 727 msec 2024-11-24T20:49:36,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-24T20:49:36,045 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:49:36,046 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T20:49:36,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] procedure2.ProcedureExecutor(1139): Stored pid=178, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:49:36,048 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=178, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:49:36,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-24T20:49:36,048 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=178, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:49:36,054 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:36,056 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A, FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B, FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C, FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/recovered.edits] 2024-11-24T20:49:36,058 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/3de5a4c21d0045ea8c92372a3a8282ba to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/3de5a4c21d0045ea8c92372a3a8282ba 2024-11-24T20:49:36,059 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/8b8634813ab74581bce180b044857fae to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/8b8634813ab74581bce180b044857fae 2024-11-24T20:49:36,060 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/ae570a9ba4ea475eb375eb181a1dfdb5 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/A/ae570a9ba4ea475eb375eb181a1dfdb5 2024-11-24T20:49:36,061 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/3bf5374916fa4cbb9129f7198b298d3c to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/3bf5374916fa4cbb9129f7198b298d3c 2024-11-24T20:49:36,062 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/ab4f249416694416b206a935fd647c0d to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/ab4f249416694416b206a935fd647c0d 2024-11-24T20:49:36,063 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/cf0d030691d8417ea1ae472a600a6f14 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/B/cf0d030691d8417ea1ae472a600a6f14 2024-11-24T20:49:36,065 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/3b781899142d440eb7c80469f5a54802 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/3b781899142d440eb7c80469f5a54802 2024-11-24T20:49:36,066 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/52329df339a2413f83041cca9da3ffc6 to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/52329df339a2413f83041cca9da3ffc6 2024-11-24T20:49:36,067 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/ad319cf16efd40f081d82a752027548e to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/C/ad319cf16efd40f081d82a752027548e 2024-11-24T20:49:36,069 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/recovered.edits/403.seqid to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb/recovered.edits/403.seqid 2024-11-24T20:49:36,069 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/default/TestAcidGuarantees/5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:36,069 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T20:49:36,069 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T20:49:36,070 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-24T20:49:36,072 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240e25f1c210f142e3a071f73342545eac_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240e25f1c210f142e3a071f73342545eac_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:36,073 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240f03a0224c554026ae7ce8035b9f8c96_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240f03a0224c554026ae7ce8035b9f8c96_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:36,074 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112418b7f784043644fb9435d9e9843efb3c_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112418b7f784043644fb9435d9e9843efb3c_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:36,075 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241de45f10ad2a409187cf2292c5fb7a77_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241de45f10ad2a409187cf2292c5fb7a77_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:36,076 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124234b80ba2c87447dac28a80a1e8b054c_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124234b80ba2c87447dac28a80a1e8b054c_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:36,077 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243e7f5eacc7ba46c0a77513d6545a585d_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243e7f5eacc7ba46c0a77513d6545a585d_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:36,078 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124450fcd18c8e14ca1a0902e84aa707691_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124450fcd18c8e14ca1a0902e84aa707691_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:36,078 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112447cdf12ddeb74944b679d6fac6e0d873_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112447cdf12ddeb74944b679d6fac6e0d873_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:36,079 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245d12f0778e2d4cfb8f01772d2624059f_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245d12f0778e2d4cfb8f01772d2624059f_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:36,080 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246ceb19ded5b74301bb5471f590a8e472_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246ceb19ded5b74301bb5471f590a8e472_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:36,081 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112474fa8fb92a1d4d0ab8993b54bdf3b1a2_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112474fa8fb92a1d4d0ab8993b54bdf3b1a2_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:36,082 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247afb4bbe6bfc43cbb57c726b9e03099b_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247afb4bbe6bfc43cbb57c726b9e03099b_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:36,083 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248339d37634fe4e81b1a491ed75699711_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248339d37634fe4e81b1a491ed75699711_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:36,084 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124891e4389b7f94a73bb2f17b01bd21951_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124891e4389b7f94a73bb2f17b01bd21951_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:36,084 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124aae2dfa38bd14e268a819a22296a2158_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124aae2dfa38bd14e268a819a22296a2158_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:36,085 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b62b6a85f5f34f8eacec5c9d1daca17b_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b62b6a85f5f34f8eacec5c9d1daca17b_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:36,086 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c23b3a040d024fbbae840c4093471080_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c23b3a040d024fbbae840c4093471080_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:36,087 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c3b08dadfef84b2ea9ad4bd0ffc195b6_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c3b08dadfef84b2ea9ad4bd0ffc195b6_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:36,088 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e9723e74272c4602a0565e3af48622a1_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e9723e74272c4602a0565e3af48622a1_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:36,088 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f291050713cc444f9ec48f6ea9bbacea_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f291050713cc444f9ec48f6ea9bbacea_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:36,089 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fa8ecf85cb31433bb1eb177d589ee2bc_5eaa0424086a20a4fd6f784698d02aeb to hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fa8ecf85cb31433bb1eb177d589ee2bc_5eaa0424086a20a4fd6f784698d02aeb 2024-11-24T20:49:36,089 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T20:49:36,091 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=178, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:49:36,093 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T20:49:36,095 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T20:49:36,096 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=178, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:49:36,096 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T20:49:36,096 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732481376096"}]},"ts":"9223372036854775807"} 2024-11-24T20:49:36,098 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T20:49:36,098 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5eaa0424086a20a4fd6f784698d02aeb, NAME => 'TestAcidGuarantees,,1732481349214.5eaa0424086a20a4fd6f784698d02aeb.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T20:49:36,098 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T20:49:36,098 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732481376098"}]},"ts":"9223372036854775807"} 2024-11-24T20:49:36,100 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T20:49:36,101 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=178, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T20:49:36,102 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=178, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 55 msec 2024-11-24T20:49:36,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-24T20:49:36,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T20:49:36,155 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T20:49:36,164 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=239 (was 236) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=518 (was 469) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=10093 (was 9154) - AvailableMemoryMB LEAK? - 2024-11-24T20:49:36,164 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T20:49:36,164 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T20:49:36,165 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T20:49:36,165 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:36,165 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:36,165 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T20:49:36,165 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T20:49:36,165 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=840105357, stopped=false 2024-11-24T20:49:36,165 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=20b981ef5c27,38339,1732481181363 2024-11-24T20:49:36,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34625-0x1016e98df0e0001, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T20:49:36,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38339-0x1016e98df0e0000, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T20:49:36,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34625-0x1016e98df0e0001, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:49:36,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38339-0x1016e98df0e0000, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:49:36,234 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T20:49:36,235 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T20:49:36,235 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T20:49:36,235 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:36,235 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34625-0x1016e98df0e0001, quorum=127.0.0.1:63551, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T20:49:36,235 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38339-0x1016e98df0e0000, quorum=127.0.0.1:63551, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T20:49:36,237 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '20b981ef5c27,34625,1732481182291' ***** 2024-11-24T20:49:36,237 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T20:49:36,237 INFO [RS:0;20b981ef5c27:34625 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T20:49:36,238 INFO [RS:0;20b981ef5c27:34625 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T20:49:36,238 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T20:49:36,238 INFO [RS:0;20b981ef5c27:34625 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T20:49:36,238 INFO [RS:0;20b981ef5c27:34625 {}] regionserver.HRegionServer(959): stopping server 20b981ef5c27,34625,1732481182291 2024-11-24T20:49:36,238 INFO [RS:0;20b981ef5c27:34625 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T20:49:36,238 INFO [RS:0;20b981ef5c27:34625 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;20b981ef5c27:34625. 2024-11-24T20:49:36,238 DEBUG [RS:0;20b981ef5c27:34625 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T20:49:36,239 DEBUG [RS:0;20b981ef5c27:34625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:36,239 INFO [RS:0;20b981ef5c27:34625 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T20:49:36,239 INFO [RS:0;20b981ef5c27:34625 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T20:49:36,239 INFO [RS:0;20b981ef5c27:34625 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T20:49:36,239 INFO [RS:0;20b981ef5c27:34625 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T20:49:36,240 INFO [RS:0;20b981ef5c27:34625 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T20:49:36,240 DEBUG [RS:0;20b981ef5c27:34625 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-24T20:49:36,240 DEBUG [RS:0;20b981ef5c27:34625 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T20:49:36,240 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T20:49:36,241 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T20:49:36,241 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T20:49:36,241 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T20:49:36,241 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T20:49:36,241 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-24T20:49:36,261 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/hbase/meta/1588230740/.tmp/info/2dac71a761aa47bfb7abef6876e77fdd is 95, key is TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150./info:/1732481218595/DeleteFamily/seqid=0 2024-11-24T20:49:36,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742470_1646 (size=6514) 2024-11-24T20:49:36,293 INFO [regionserver/20b981ef5c27:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T20:49:36,441 DEBUG [RS:0;20b981ef5c27:34625 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T20:49:36,476 INFO [regionserver/20b981ef5c27:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-24T20:49:36,476 INFO [regionserver/20b981ef5c27:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-24T20:49:36,641 DEBUG [RS:0;20b981ef5c27:34625 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T20:49:36,664 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/hbase/meta/1588230740/.tmp/info/2dac71a761aa47bfb7abef6876e77fdd 2024-11-24T20:49:36,682 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/hbase/meta/1588230740/.tmp/ns/02891f791bd641209813daba5034f541 is 93, key is TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150./ns:/1732481218553/DeleteFamily/seqid=0 2024-11-24T20:49:36,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742471_1647 (size=5906) 2024-11-24T20:49:36,841 DEBUG [RS:0;20b981ef5c27:34625 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T20:49:37,041 DEBUG [RS:0;20b981ef5c27:34625 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T20:49:37,085 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/hbase/meta/1588230740/.tmp/ns/02891f791bd641209813daba5034f541 2024-11-24T20:49:37,108 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/hbase/meta/1588230740/.tmp/rep_barrier/92189f984e0642d087ddadf3b078a873 is 102, key is TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150./rep_barrier:/1732481218553/DeleteFamily/seqid=0 2024-11-24T20:49:37,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742472_1648 (size=6025) 2024-11-24T20:49:37,242 DEBUG [RS:0;20b981ef5c27:34625 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T20:49:37,442 DEBUG [RS:0;20b981ef5c27:34625 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T20:49:37,512 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/hbase/meta/1588230740/.tmp/rep_barrier/92189f984e0642d087ddadf3b078a873 2024-11-24T20:49:37,534 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/hbase/meta/1588230740/.tmp/table/f7ea760006f4467bb86b9b30a77ec32e is 96, key is TestAcidGuarantees,,1732481185821.6ac72b5882ff154796d9a89886bf5150./table:/1732481218553/DeleteFamily/seqid=0 2024-11-24T20:49:37,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742473_1649 (size=5936) 2024-11-24T20:49:37,542 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/hbase/meta/1588230740/.tmp/table/f7ea760006f4467bb86b9b30a77ec32e 2024-11-24T20:49:37,545 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/hbase/meta/1588230740/.tmp/info/2dac71a761aa47bfb7abef6876e77fdd as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/hbase/meta/1588230740/info/2dac71a761aa47bfb7abef6876e77fdd 2024-11-24T20:49:37,548 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/hbase/meta/1588230740/info/2dac71a761aa47bfb7abef6876e77fdd, entries=12, sequenceid=90, filesize=6.4 K 2024-11-24T20:49:37,549 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/hbase/meta/1588230740/.tmp/ns/02891f791bd641209813daba5034f541 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/hbase/meta/1588230740/ns/02891f791bd641209813daba5034f541 2024-11-24T20:49:37,552 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/hbase/meta/1588230740/ns/02891f791bd641209813daba5034f541, entries=8, sequenceid=90, filesize=5.8 K 2024-11-24T20:49:37,552 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/hbase/meta/1588230740/.tmp/rep_barrier/92189f984e0642d087ddadf3b078a873 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/hbase/meta/1588230740/rep_barrier/92189f984e0642d087ddadf3b078a873 2024-11-24T20:49:37,556 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/hbase/meta/1588230740/rep_barrier/92189f984e0642d087ddadf3b078a873, entries=6, sequenceid=90, filesize=5.9 K 2024-11-24T20:49:37,556 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/hbase/meta/1588230740/.tmp/table/f7ea760006f4467bb86b9b30a77ec32e as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/hbase/meta/1588230740/table/f7ea760006f4467bb86b9b30a77ec32e 2024-11-24T20:49:37,560 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/hbase/meta/1588230740/table/f7ea760006f4467bb86b9b30a77ec32e, entries=7, sequenceid=90, filesize=5.8 K 2024-11-24T20:49:37,560 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1319ms, sequenceid=90, compaction requested=false 2024-11-24T20:49:37,567 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-24T20:49:37,567 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T20:49:37,567 INFO [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T20:49:37,567 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732481376240Running coprocessor pre-close hooks at 1732481376240Disabling compacts and flushes for region at 1732481376240Disabling writes for close at 1732481376241 (+1 ms)Obtaining lock to block concurrent updates at 1732481376241Preparing flush snapshotting stores in 1588230740 at 1732481376241Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732481376241Flushing stores of hbase:meta,,1.1588230740 at 1732481376242 (+1 ms)Flushing 1588230740/info: creating writer at 1732481376242Flushing 1588230740/info: appending metadata at 1732481376261 (+19 ms)Flushing 1588230740/info: closing flushed file at 1732481376261Flushing 1588230740/ns: creating writer at 1732481376668 (+407 ms)Flushing 1588230740/ns: appending metadata at 1732481376681 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1732481376681Flushing 1588230740/rep_barrier: creating writer at 1732481377089 (+408 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732481377108 (+19 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732481377108Flushing 1588230740/table: creating writer at 1732481377515 (+407 ms)Flushing 1588230740/table: appending metadata at 1732481377534 (+19 ms)Flushing 1588230740/table: closing flushed file at 1732481377534Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d7071ea: reopening flushed file at 1732481377545 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@220ef323: reopening flushed file at 1732481377548 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b49b78b: reopening flushed file at 1732481377552 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@417e6a3e: reopening flushed file at 1732481377556 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1319ms, sequenceid=90, compaction requested=false at 1732481377560 (+4 ms)Writing region close event to WAL at 1732481377564 (+4 ms)Running coprocessor post-close hooks at 1732481377567 (+3 ms)Closed at 1732481377567 2024-11-24T20:49:37,568 DEBUG [RS_CLOSE_META-regionserver/20b981ef5c27:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T20:49:37,642 INFO [RS:0;20b981ef5c27:34625 {}] regionserver.HRegionServer(976): stopping server 20b981ef5c27,34625,1732481182291; all regions closed. 2024-11-24T20:49:37,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741834_1010 (size=25344) 2024-11-24T20:49:37,648 DEBUG [RS:0;20b981ef5c27:34625 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/oldWALs 2024-11-24T20:49:37,648 INFO [RS:0;20b981ef5c27:34625 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 20b981ef5c27%2C34625%2C1732481182291.meta:.meta(num 1732481185218) 2024-11-24T20:49:37,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741833_1009 (size=15011639) 2024-11-24T20:49:37,652 DEBUG [RS:0;20b981ef5c27:34625 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/oldWALs 2024-11-24T20:49:37,652 INFO [RS:0;20b981ef5c27:34625 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 20b981ef5c27%2C34625%2C1732481182291:(num 1732481184647) 2024-11-24T20:49:37,652 DEBUG [RS:0;20b981ef5c27:34625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T20:49:37,652 INFO [RS:0;20b981ef5c27:34625 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T20:49:37,652 INFO [RS:0;20b981ef5c27:34625 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T20:49:37,652 INFO [RS:0;20b981ef5c27:34625 {}] hbase.ChoreService(370): Chore service for: regionserver/20b981ef5c27:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T20:49:37,652 INFO [RS:0;20b981ef5c27:34625 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T20:49:37,652 INFO [regionserver/20b981ef5c27:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T20:49:37,653 INFO [RS:0;20b981ef5c27:34625 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34625 2024-11-24T20:49:37,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38339-0x1016e98df0e0000, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T20:49:37,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34625-0x1016e98df0e0001, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/20b981ef5c27,34625,1732481182291 2024-11-24T20:49:37,675 INFO [RS:0;20b981ef5c27:34625 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T20:49:37,686 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [20b981ef5c27,34625,1732481182291] 2024-11-24T20:49:37,696 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/20b981ef5c27,34625,1732481182291 already deleted, retry=false 2024-11-24T20:49:37,696 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 20b981ef5c27,34625,1732481182291 expired; onlineServers=0 2024-11-24T20:49:37,697 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '20b981ef5c27,38339,1732481181363' ***** 2024-11-24T20:49:37,697 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T20:49:37,697 INFO [M:0;20b981ef5c27:38339 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T20:49:37,697 INFO [M:0;20b981ef5c27:38339 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T20:49:37,697 DEBUG [M:0;20b981ef5c27:38339 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T20:49:37,697 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T20:49:37,697 DEBUG [M:0;20b981ef5c27:38339 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T20:49:37,697 DEBUG [master/20b981ef5c27:0:becomeActiveMaster-HFileCleaner.large.0-1732481184309 {}] cleaner.HFileCleaner(306): Exit Thread[master/20b981ef5c27:0:becomeActiveMaster-HFileCleaner.large.0-1732481184309,5,FailOnTimeoutGroup] 2024-11-24T20:49:37,697 DEBUG [master/20b981ef5c27:0:becomeActiveMaster-HFileCleaner.small.0-1732481184310 {}] cleaner.HFileCleaner(306): Exit Thread[master/20b981ef5c27:0:becomeActiveMaster-HFileCleaner.small.0-1732481184310,5,FailOnTimeoutGroup] 2024-11-24T20:49:37,698 INFO [M:0;20b981ef5c27:38339 {}] hbase.ChoreService(370): Chore service for: master/20b981ef5c27:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T20:49:37,698 INFO [M:0;20b981ef5c27:38339 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T20:49:37,698 DEBUG [M:0;20b981ef5c27:38339 {}] master.HMaster(1795): Stopping service threads 2024-11-24T20:49:37,698 INFO [M:0;20b981ef5c27:38339 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T20:49:37,698 INFO [M:0;20b981ef5c27:38339 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T20:49:37,698 ERROR [M:0;20b981ef5c27:38339 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[IPC Client (726517346) connection to localhost/127.0.0.1:45873 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:45873,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-24T20:49:37,699 INFO [M:0;20b981ef5c27:38339 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T20:49:37,699 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T20:49:37,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38339-0x1016e98df0e0000, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T20:49:37,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38339-0x1016e98df0e0000, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T20:49:37,707 DEBUG [M:0;20b981ef5c27:38339 {}] zookeeper.ZKUtil(347): master:38339-0x1016e98df0e0000, quorum=127.0.0.1:63551, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T20:49:37,707 WARN [M:0;20b981ef5c27:38339 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T20:49:37,708 INFO [M:0;20b981ef5c27:38339 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/.lastflushedseqids 2024-11-24T20:49:37,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742474_1650 (size=72) 2024-11-24T20:49:37,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34625-0x1016e98df0e0001, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T20:49:37,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34625-0x1016e98df0e0001, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T20:49:37,787 INFO [RS:0;20b981ef5c27:34625 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T20:49:37,787 INFO [RS:0;20b981ef5c27:34625 {}] regionserver.HRegionServer(1031): Exiting; stopping=20b981ef5c27,34625,1732481182291; zookeeper connection closed. 2024-11-24T20:49:37,787 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@18e00b4b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@18e00b4b 2024-11-24T20:49:37,788 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-24T20:49:38,118 INFO [M:0;20b981ef5c27:38339 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T20:49:38,119 INFO [M:0;20b981ef5c27:38339 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T20:49:38,119 DEBUG [M:0;20b981ef5c27:38339 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T20:49:38,119 INFO [M:0;20b981ef5c27:38339 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T20:49:38,119 DEBUG [M:0;20b981ef5c27:38339 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T20:49:38,119 DEBUG [M:0;20b981ef5c27:38339 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T20:49:38,119 DEBUG [M:0;20b981ef5c27:38339 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T20:49:38,119 INFO [M:0;20b981ef5c27:38339 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=756.66 KB heapSize=929.84 KB 2024-11-24T20:49:38,139 DEBUG [M:0;20b981ef5c27:38339 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/13605aabdb014c789f62fe702065b2a8 is 82, key is hbase:meta,,1/info:regioninfo/1732481185310/Put/seqid=0 2024-11-24T20:49:38,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742475_1651 (size=5672) 2024-11-24T20:49:38,142 INFO [M:0;20b981ef5c27:38339 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2130 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/13605aabdb014c789f62fe702065b2a8 2024-11-24T20:49:38,161 DEBUG [M:0;20b981ef5c27:38339 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/80470d15900b4360a266201f2a750235 is 2276, key is \x00\x00\x00\x00\x00\x00\x00\x95/proc:d/1732481352302/Put/seqid=0 2024-11-24T20:49:38,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742476_1652 (size=45522) 2024-11-24T20:49:38,566 INFO [M:0;20b981ef5c27:38339 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=756.10 KB at sequenceid=2130 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/80470d15900b4360a266201f2a750235 2024-11-24T20:49:38,570 INFO [M:0;20b981ef5c27:38339 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 80470d15900b4360a266201f2a750235 2024-11-24T20:49:38,589 DEBUG [M:0;20b981ef5c27:38339 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/532c724eda744ec082f0526cfb0c1588 is 69, key is 20b981ef5c27,34625,1732481182291/rs:state/1732481184369/Put/seqid=0 2024-11-24T20:49:38,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073742477_1653 (size=5156) 2024-11-24T20:49:38,992 INFO [M:0;20b981ef5c27:38339 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2130 (bloomFilter=true), to=hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/532c724eda744ec082f0526cfb0c1588 2024-11-24T20:49:38,998 DEBUG [M:0;20b981ef5c27:38339 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/13605aabdb014c789f62fe702065b2a8 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/13605aabdb014c789f62fe702065b2a8 2024-11-24T20:49:39,002 INFO [M:0;20b981ef5c27:38339 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/13605aabdb014c789f62fe702065b2a8, entries=8, sequenceid=2130, filesize=5.5 K 2024-11-24T20:49:39,002 DEBUG [M:0;20b981ef5c27:38339 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/80470d15900b4360a266201f2a750235 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/80470d15900b4360a266201f2a750235 2024-11-24T20:49:39,006 INFO [M:0;20b981ef5c27:38339 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 80470d15900b4360a266201f2a750235 2024-11-24T20:49:39,006 INFO [M:0;20b981ef5c27:38339 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/80470d15900b4360a266201f2a750235, entries=178, sequenceid=2130, filesize=44.5 K 2024-11-24T20:49:39,007 DEBUG [M:0;20b981ef5c27:38339 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/532c724eda744ec082f0526cfb0c1588 as hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/532c724eda744ec082f0526cfb0c1588 2024-11-24T20:49:39,010 INFO [M:0;20b981ef5c27:38339 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45873/user/jenkins/test-data/b1490709-54b9-74e2-1fde-7bc84b6c094a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/532c724eda744ec082f0526cfb0c1588, entries=1, sequenceid=2130, filesize=5.0 K 2024-11-24T20:49:39,010 INFO [M:0;20b981ef5c27:38339 {}] regionserver.HRegion(3140): Finished flush of dataSize ~756.66 KB/774817, heapSize ~929.54 KB/951848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 891ms, sequenceid=2130, compaction requested=false 2024-11-24T20:49:39,011 INFO [M:0;20b981ef5c27:38339 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T20:49:39,011 DEBUG [M:0;20b981ef5c27:38339 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732481378119Disabling compacts and flushes for region at 1732481378119Disabling writes for close at 1732481378119Obtaining lock to block concurrent updates at 1732481378119Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732481378119Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=774817, getHeapSize=952088, getOffHeapSize=0, getCellsCount=2333 at 1732481378119Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732481378120 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732481378120Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732481378138 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732481378138Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732481378145 (+7 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732481378161 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732481378161Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732481378570 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732481378588 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732481378588Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6da8bc50: reopening flushed file at 1732481378997 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1744881: reopening flushed file at 1732481379002 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@200a8d81: reopening flushed file at 1732481379007 (+5 ms)Finished flush of dataSize ~756.66 KB/774817, heapSize ~929.54 KB/951848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 891ms, sequenceid=2130, compaction requested=false at 1732481379010 (+3 ms)Writing region close event to WAL at 1732481379011 (+1 ms)Closed at 1732481379011 2024-11-24T20:49:39,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40053 is added to blk_1073741830_1006 (size=914498) 2024-11-24T20:49:39,014 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T20:49:39,014 INFO [M:0;20b981ef5c27:38339 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T20:49:39,014 INFO [M:0;20b981ef5c27:38339 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38339 2024-11-24T20:49:39,015 INFO [M:0;20b981ef5c27:38339 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T20:49:39,171 INFO [M:0;20b981ef5c27:38339 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T20:49:39,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38339-0x1016e98df0e0000, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T20:49:39,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38339-0x1016e98df0e0000, quorum=127.0.0.1:63551, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T20:49:39,175 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3dfb1097{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T20:49:39,177 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@16142ff{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T20:49:39,177 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T20:49:39,177 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ef5f873{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T20:49:39,177 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b8e8cdd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/hadoop.log.dir/,STOPPED} 2024-11-24T20:49:39,180 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T20:49:39,180 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T20:49:39,180 WARN [BP-171709779-172.17.0.2-1732481177570 heartbeating to localhost/127.0.0.1:45873 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T20:49:39,180 WARN [BP-171709779-172.17.0.2-1732481177570 heartbeating to localhost/127.0.0.1:45873 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-171709779-172.17.0.2-1732481177570 (Datanode Uuid fd6f659b-0aaf-4b1f-84d2-c3f2d7a13354) service to localhost/127.0.0.1:45873 2024-11-24T20:49:39,184 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/cluster_32fb3c48-f3a2-7f88-64a5-636927e4ea42/data/data1/current/BP-171709779-172.17.0.2-1732481177570 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T20:49:39,184 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/cluster_32fb3c48-f3a2-7f88-64a5-636927e4ea42/data/data2/current/BP-171709779-172.17.0.2-1732481177570 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T20:49:39,185 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T20:49:39,191 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bbf242f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T20:49:39,192 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T20:49:39,192 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T20:49:39,192 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T20:49:39,192 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a6f68163-35ef-a2b9-cb0f-a16d11b7c45c/hadoop.log.dir/,STOPPED} 2024-11-24T20:49:39,212 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T20:49:39,333 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down